Class: org.apache.hadoop.fs.TestChecksumFileSystem

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testCorruptedChecksum() throws Exception {
  Path testPath=new Path(TEST_ROOT_DIR,"testCorruptChecksum");
  Path checksumPath=localFs.getChecksumFile(testPath);
  FSDataOutputStream out=localFs.create(testPath,true);
  out.write("testing 1 2 3".getBytes());
  out.close();
  assertTrue(localFs.exists(checksumPath));
  FileStatus stat=localFs.getFileStatus(checksumPath);
  out=localFs.getRawFileSystem().create(testPath,true);
  out.write("testing stale checksum".getBytes());
  out.close();
  assertTrue(localFs.exists(checksumPath));
  assertEquals(stat,localFs.getFileStatus(checksumPath));
  Exception e=null;
  try {
    localFs.setVerifyChecksum(true);
    readFile(localFs,testPath,1024);
  }
 catch (  ChecksumException ce) {
    e=ce;
  }
 finally {
    assertNotNull("got checksum error",e);
  }
  localFs.setVerifyChecksum(false);
  String str=readFile(localFs,testPath,1024);
  assertEquals("testing stale checksum",str);
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * Test to ensure that if the checksum file is truncated, a
 * ChecksumException is thrown
 */
@Test public void testTruncatedChecksum() throws Exception {
  Path testPath=new Path(TEST_ROOT_DIR,"testtruncatedcrc");
  FSDataOutputStream fout=localFs.create(testPath);
  fout.write("testing truncation".getBytes());
  fout.close();
  Path checksumFile=localFs.getChecksumFile(testPath);
  FileSystem rawFs=localFs.getRawFileSystem();
  FSDataInputStream checksumStream=rawFs.open(checksumFile);
  byte buf[]=new byte[8192];
  int read=checksumStream.read(buf,0,buf.length);
  checksumStream.close();
  FSDataOutputStream replaceStream=rawFs.create(checksumFile);
  replaceStream.write(buf,0,read - 1);
  replaceStream.close();
  try {
    readFile(localFs,testPath,1024);
    fail("Did not throw a ChecksumException when reading truncated " + "crc file");
  }
 catch (  ChecksumException ie) {
  }
  localFs.setVerifyChecksum(false);
  String str=readFile(localFs,testPath,1024).toString();
  assertTrue("read","testing truncation".equals(str));
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testVerifyChecksum() throws Exception {
  Path testPath=new Path(TEST_ROOT_DIR,"testPath");
  Path testPath11=new Path(TEST_ROOT_DIR,"testPath11");
  FSDataOutputStream fout=localFs.create(testPath);
  fout.write("testing".getBytes());
  fout.close();
  fout=localFs.create(testPath11);
  fout.write("testing you".getBytes());
  fout.close();
  readFile(localFs,testPath,128);
  readFile(localFs,testPath,511);
  readFile(localFs,testPath,512);
  readFile(localFs,testPath,513);
  readFile(localFs,testPath,1023);
  readFile(localFs,testPath,1024);
  readFile(localFs,testPath,1025);
  localFs.delete(localFs.getChecksumFile(testPath),true);
  assertTrue("checksum deleted",!localFs.exists(localFs.getChecksumFile(testPath)));
  FileUtil.copy(localFs,localFs.getChecksumFile(testPath11),localFs,localFs.getChecksumFile(testPath),false,true,localFs.getConf());
  assertTrue("checksum exists",localFs.exists(localFs.getChecksumFile(testPath)));
  boolean errorRead=false;
  try {
    readFile(localFs,testPath,1024);
  }
 catch (  ChecksumException ie) {
    errorRead=true;
  }
  assertTrue("error reading",errorRead);
  localFs.setVerifyChecksum(false);
  String str=readFile(localFs,testPath,1024).toString();
  assertTrue("read","testing".equals(str));
}

Class: org.apache.hadoop.fs.TestDFVariations

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=5000) public void testGetMountCurrentDirectory() throws Exception {
  File currentDirectory=new File(".");
  String workingDir=currentDirectory.getAbsoluteFile().getCanonicalPath();
  DF df=new DF(new File(workingDir),0L);
  String mountPath=df.getMount();
  File mountDir=new File(mountPath);
  assertTrue("Mount dir [" + mountDir.getAbsolutePath() + "] should exist.",mountDir.exists());
  assertTrue("Mount dir [" + mountDir.getAbsolutePath() + "] should be directory.",mountDir.isDirectory());
  assertTrue("Working dir [" + workingDir + "] should start with ["+ mountPath+ "].",workingDir.startsWith(mountPath));
}

Class: org.apache.hadoop.fs.TestEnhancedByteBufferAccess

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier HybridVerifier 

@Test public void testZeroCopyMmapCache() throws Exception {
  HdfsConfiguration conf=initZeroCopyTest();
  MiniDFSCluster cluster=null;
  final Path TEST_PATH=new Path("/a");
  final int TEST_FILE_LENGTH=16385;
  final int RANDOM_SEED=23453;
  final String CONTEXT="testZeroCopyMmapCacheContext";
  FSDataInputStream fsIn=null;
  ByteBuffer results[]={null,null,null,null};
  DistributedFileSystem fs=null;
  conf.set(DFSConfigKeys.DFS_CLIENT_CONTEXT,CONTEXT);
  cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  fs=cluster.getFileSystem();
  DFSTestUtil.createFile(fs,TEST_PATH,TEST_FILE_LENGTH,(short)1,RANDOM_SEED);
  try {
    DFSTestUtil.waitReplication(fs,TEST_PATH,(short)1);
  }
 catch (  InterruptedException e) {
    Assert.fail("unexpected InterruptedException during " + "waitReplication: " + e);
  }
catch (  TimeoutException e) {
    Assert.fail("unexpected TimeoutException during " + "waitReplication: " + e);
  }
  fsIn=fs.open(TEST_PATH);
  byte original[]=new byte[TEST_FILE_LENGTH];
  IOUtils.readFully(fsIn,original,0,TEST_FILE_LENGTH);
  fsIn.close();
  fsIn=fs.open(TEST_PATH);
  final ShortCircuitCache cache=ClientContext.get(CONTEXT,new DFSClient.Conf(conf)).getShortCircuitCache();
  cache.accept(new CountingVisitor(0,5,5,0));
  results[0]=fsIn.read(null,4096,EnumSet.of(ReadOption.SKIP_CHECKSUMS));
  fsIn.seek(0);
  results[1]=fsIn.read(null,4096,EnumSet.of(ReadOption.SKIP_CHECKSUMS));
  final ExtendedBlock firstBlock=DFSTestUtil.getFirstBlock(fs,TEST_PATH);
  cache.accept(new CacheVisitor(){
    @Override public void visit(    int numOutstandingMmaps,    Map replicas,    Map failedLoads,    Map evictable,    Map evictableMmapped){
      ShortCircuitReplica replica=replicas.get(new ExtendedBlockId(firstBlock.getBlockId(),firstBlock.getBlockPoolId()));
      Assert.assertNotNull(replica);
      Assert.assertTrue(replica.hasMmap());
      Assert.assertNull(replica.getEvictableTimeNs());
    }
  }
);
  results[2]=fsIn.read(null,4096,EnumSet.of(ReadOption.SKIP_CHECKSUMS));
  results[3]=fsIn.read(null,4096,EnumSet.of(ReadOption.SKIP_CHECKSUMS));
  cache.accept(new CountingVisitor(3,5,2,0));
  for (  ByteBuffer buffer : results) {
    if (buffer != null) {
      fsIn.releaseBuffer(buffer);
    }
  }
  fsIn.close();
  GenericTestUtils.waitFor(new Supplier(){
    public Boolean get(){
      final MutableBoolean finished=new MutableBoolean(false);
      cache.accept(new CacheVisitor(){
        @Override public void visit(        int numOutstandingMmaps,        Map replicas,        Map failedLoads,        Map evictable,        Map evictableMmapped){
          finished.setValue(evictableMmapped.isEmpty());
        }
      }
);
      return finished.booleanValue();
    }
  }
,10,60000);
  cache.accept(new CountingVisitor(0,-1,-1,-1));
  fs.close();
  cluster.shutdown();
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that we can zero-copy read cached data even without disabling
 * checksums.
 */
@Test(timeout=120000) public void testZeroCopyReadOfCachedData() throws Exception {
  BlockReaderTestUtil.enableShortCircuitShmTracing();
  BlockReaderTestUtil.enableBlockReaderFactoryTracing();
  BlockReaderTestUtil.enableHdfsCachingTracing();
  final int TEST_FILE_LENGTH=16385;
  final Path TEST_PATH=new Path("/a");
  final int RANDOM_SEED=23453;
  HdfsConfiguration conf=initZeroCopyTest();
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY,false);
  final String CONTEXT="testZeroCopyReadOfCachedData";
  conf.set(DFSConfigKeys.DFS_CLIENT_CONTEXT,CONTEXT);
  conf.setLong(DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,DFSTestUtil.roundUpToMultiple(TEST_FILE_LENGTH,4096));
  MiniDFSCluster cluster=null;
  ByteBuffer result=null, result2=null;
  cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  FsDatasetSpi fsd=cluster.getDataNodes().get(0).getFSDataset();
  DistributedFileSystem fs=cluster.getFileSystem();
  DFSTestUtil.createFile(fs,TEST_PATH,TEST_FILE_LENGTH,(short)1,RANDOM_SEED);
  DFSTestUtil.waitReplication(fs,TEST_PATH,(short)1);
  byte original[]=DFSTestUtil.calculateFileContentsFromSeed(RANDOM_SEED,TEST_FILE_LENGTH);
  FSDataInputStream fsIn=fs.open(TEST_PATH);
  try {
    result=fsIn.read(null,TEST_FILE_LENGTH / 2,EnumSet.noneOf(ReadOption.class));
    Assert.fail("expected UnsupportedOperationException");
  }
 catch (  UnsupportedOperationException e) {
  }
  fs.addCachePool(new CachePoolInfo("pool1"));
  long directiveId=fs.addCacheDirective(new CacheDirectiveInfo.Builder().setPath(TEST_PATH).setReplication((short)1).setPool("pool1").build());
  int numBlocks=(int)Math.ceil((double)TEST_FILE_LENGTH / BLOCK_SIZE);
  DFSTestUtil.verifyExpectedCacheUsage(DFSTestUtil.roundUpToMultiple(TEST_FILE_LENGTH,BLOCK_SIZE),numBlocks,cluster.getDataNodes().get(0).getFSDataset());
  try {
    result=fsIn.read(null,TEST_FILE_LENGTH,EnumSet.noneOf(ReadOption.class));
  }
 catch (  UnsupportedOperationException e) {
    Assert.fail("expected to be able to read cached file via zero-copy");
  }
  Assert.assertArrayEquals(Arrays.copyOfRange(original,0,BLOCK_SIZE),byteBufferToArray(result));
  FSDataInputStream fsIn2=fs.open(TEST_PATH);
  try {
    result2=fsIn2.read(null,TEST_FILE_LENGTH,EnumSet.noneOf(ReadOption.class));
  }
 catch (  UnsupportedOperationException e) {
    Assert.fail("expected to be able to read cached file via zero-copy");
  }
  Assert.assertArrayEquals(Arrays.copyOfRange(original,0,BLOCK_SIZE),byteBufferToArray(result2));
  fsIn2.releaseBuffer(result2);
  fsIn2.close();
  final ExtendedBlock firstBlock=DFSTestUtil.getFirstBlock(fs,TEST_PATH);
  final ShortCircuitCache cache=ClientContext.get(CONTEXT,new DFSClient.Conf(conf)).getShortCircuitCache();
  waitForReplicaAnchorStatus(cache,firstBlock,true,true,1);
  fs.removeCacheDirective(directiveId);
  waitForReplicaAnchorStatus(cache,firstBlock,false,true,1);
  fsIn.releaseBuffer(result);
  waitForReplicaAnchorStatus(cache,firstBlock,false,false,1);
  DFSTestUtil.verifyExpectedCacheUsage(0,0,fsd);
  fsIn.close();
  fs.close();
  cluster.shutdown();
}

Class: org.apache.hadoop.fs.TestFileContextResolveAfs

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=30000) public void testFileContextResolveAfs() throws IOException {
  Configuration conf=new Configuration();
  localFs=FileSystem.get(conf);
  Path localPath=new Path(TEST_ROOT_DIR_LOCAL + "/TestFileContextResolveAfs1");
  Path linkPath=localFs.makeQualified(new Path(TEST_ROOT_DIR_LOCAL,"TestFileContextResolveAfs2"));
  localFs.mkdirs(new Path(TEST_ROOT_DIR_LOCAL));
  localFs.create(localPath);
  fc.createSymlink(localPath,linkPath,true);
  Set afsList=fc.resolveAbstractFileSystems(linkPath);
  Assert.assertEquals(1,afsList.size());
  localFs.deleteOnExit(localPath);
  localFs.deleteOnExit(linkPath);
  localFs.close();
}

Class: org.apache.hadoop.fs.TestFileSystemCaching

APIUtilityVerifier UtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier HybridVerifier 

@Test public void testDefaultFsUris() throws Exception {
  final Configuration conf=new Configuration();
  conf.set("fs.defaultfs.impl",DefaultFs.class.getName());
  final URI defaultUri=URI.create("defaultfs://host");
  FileSystem.setDefaultUri(conf,defaultUri);
  FileSystem fs=null;
  final FileSystem defaultFs=FileSystem.get(conf);
  assertEquals(defaultUri,defaultFs.getUri());
  fs=FileSystem.get(URI.create("defaultfs:/"),conf);
  assertSame(defaultFs,fs);
  fs=FileSystem.get(URI.create("defaultfs:///"),conf);
  assertSame(defaultFs,fs);
  fs=FileSystem.get(URI.create("defaultfs://host"),conf);
  assertSame(defaultFs,fs);
  fs=FileSystem.get(URI.create("defaultfs://host2"),conf);
  assertNotSame(defaultFs,fs);
  fs=FileSystem.get(URI.create("/"),conf);
  assertSame(defaultFs,fs);
  try {
    fs=FileSystem.get(URI.create("//host"),conf);
    fail("got fs with auth but no scheme");
  }
 catch (  Exception e) {
    assertEquals("No FileSystem for scheme: null",e.getMessage());
  }
  try {
    fs=FileSystem.get(URI.create("//host2"),conf);
    fail("got fs with auth but no scheme");
  }
 catch (  Exception e) {
    assertEquals("No FileSystem for scheme: null",e.getMessage());
  }
}

APIUtilityVerifier InternalCallVerifier IdentityVerifier 

@Test public void testUserFS() throws Exception {
  final Configuration conf=new Configuration();
  conf.set("fs.cachedfile.impl",FileSystem.getFileSystemClass("file",null).getName());
  FileSystem fsU1=FileSystem.get(new URI("cachedfile://a"),conf,"bar");
  FileSystem fsU2=FileSystem.get(new URI("cachedfile://a"),conf,"foo");
  assertNotSame(fsU1,fsU2);
}

APIUtilityVerifier InternalCallVerifier IdentityVerifier 

@SuppressWarnings("unchecked") @Test public void testCacheForUgi() throws Exception {
  final Configuration conf=new Configuration();
  conf.set("fs.cachedfile.impl",FileSystem.getFileSystemClass("file",null).getName());
  UserGroupInformation ugiA=UserGroupInformation.createRemoteUser("foo");
  UserGroupInformation ugiB=UserGroupInformation.createRemoteUser("bar");
  FileSystem fsA=ugiA.doAs(new PrivilegedExceptionAction(){
    @Override public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"),conf);
    }
  }
);
  FileSystem fsA1=ugiA.doAs(new PrivilegedExceptionAction(){
    @Override public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"),conf);
    }
  }
);
  assertSame(fsA,fsA1);
  FileSystem fsB=ugiB.doAs(new PrivilegedExceptionAction(){
    @Override public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"),conf);
    }
  }
);
  assertNotSame(fsA,fsB);
  Token t1=mock(Token.class);
  UserGroupInformation ugiA2=UserGroupInformation.createRemoteUser("foo");
  fsA=ugiA2.doAs(new PrivilegedExceptionAction(){
    @Override public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"),conf);
    }
  }
);
  assertNotSame(fsA,fsA1);
  ugiA.addToken(t1);
  fsA=ugiA.doAs(new PrivilegedExceptionAction(){
    @Override public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"),conf);
    }
  }
);
  assertSame(fsA,fsA1);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testFsUniqueness() throws Exception {
  final Configuration conf=new Configuration();
  conf.set("fs.cachedfile.impl",FileSystem.getFileSystemClass("file",null).getName());
  FileSystem fs1=FileSystem.get(conf);
  FileSystem fs2=FileSystem.get(conf);
  assertTrue(fs1 == fs2);
  fs1=FileSystem.newInstance(new URI("cachedfile://a"),conf,"bar");
  fs2=FileSystem.newInstance(new URI("cachedfile://a"),conf,"bar");
  assertTrue(fs1 != fs2 && !fs1.equals(fs2));
  fs1.close();
  fs2.close();
}

APIUtilityVerifier InternalCallVerifier IdentityVerifier 

@Test public void testCloseAllForUGI() throws Exception {
  final Configuration conf=new Configuration();
  conf.set("fs.cachedfile.impl",FileSystem.getFileSystemClass("file",null).getName());
  UserGroupInformation ugiA=UserGroupInformation.createRemoteUser("foo");
  FileSystem fsA=ugiA.doAs(new PrivilegedExceptionAction(){
    @Override public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"),conf);
    }
  }
);
  FileSystem fsA1=ugiA.doAs(new PrivilegedExceptionAction(){
    @Override public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"),conf);
    }
  }
);
  assertSame(fsA,fsA1);
  FileSystem.closeAllForUGI(ugiA);
  fsA1=ugiA.doAs(new PrivilegedExceptionAction(){
    @Override public FileSystem run() throws Exception {
      return FileSystem.get(new URI("cachedfile://a"),conf);
    }
  }
);
  assertNotSame(fsA,fsA1);
}

APIUtilityVerifier InternalCallVerifier IdentityVerifier 

@Test public void testCacheDisabled() throws Exception {
  Configuration conf=new Configuration();
  conf.set("fs.uncachedfile.impl",FileSystem.getFileSystemClass("file",null).getName());
  conf.setBoolean("fs.uncachedfile.impl.disable.cache",true);
  FileSystem fs1=FileSystem.get(new URI("uncachedfile://a"),conf);
  FileSystem fs2=FileSystem.get(new URI("uncachedfile://a"),conf);
  assertNotSame(fs1,fs2);
}

APIUtilityVerifier InternalCallVerifier IdentityVerifier 

@Test public void testCacheEnabled() throws Exception {
  Configuration conf=new Configuration();
  conf.set("fs.cachedfile.impl",FileSystem.getFileSystemClass("file",null).getName());
  FileSystem fs1=FileSystem.get(new URI("cachedfile://a"),conf);
  FileSystem fs2=FileSystem.get(new URI("cachedfile://a"),conf);
  assertSame(fs1,fs2);
}

Class: org.apache.hadoop.fs.TestFileUtil

APIUtilityVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testListAPI() throws IOException {
  setupDirs();
  String[] files=FileUtil.list(partitioned);
  Assert.assertEquals("Unexpected number of pre-existing files",2,files.length);
  File newDir=new File(tmp.getPath(),"test");
  newDir.mkdir();
  Assert.assertTrue("Failed to create test dir",newDir.exists());
  files=FileUtil.list(newDir);
  Assert.assertEquals("New directory unexpectedly contains files",0,files.length);
  newDir.delete();
  Assert.assertFalse("Failed to delete test dir",newDir.exists());
  try {
    files=FileUtil.list(newDir);
    Assert.fail("IOException expected on list() for non-existent dir " + newDir.toString());
  }
 catch (  IOException ioe) {
  }
}

APIUtilityVerifier BooleanVerifier 

@Test(timeout=30000) public void testFullyDelete() throws IOException {
  setupDirs();
  boolean ret=FileUtil.fullyDelete(del);
  Assert.assertTrue(ret);
  Assert.assertFalse(del.exists());
  validateTmpDir();
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests if fullyDelete deletes
 * (a) symlink to file only and not the file pointed to by symlink.
 * (b) symlink to dir only and not the dir pointed to by symlink.
 * @throws IOException
 */
@Test(timeout=30000) public void testFullyDeleteSymlinks() throws IOException {
  setupDirs();
  File link=new File(del,LINK);
  Assert.assertEquals(5,del.list().length);
  boolean ret=FileUtil.fullyDelete(link);
  Assert.assertTrue(ret);
  Assert.assertFalse(link.exists());
  Assert.assertEquals(4,del.list().length);
  validateTmpDir();
  File linkDir=new File(del,"tmpDir");
  ret=FileUtil.fullyDelete(linkDir);
  Assert.assertTrue(ret);
  Assert.assertFalse(linkDir.exists());
  Assert.assertEquals(3,del.list().length);
  validateTmpDir();
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testStat2Paths2(){
  Path defaultPath=new Path("file://default");
  Path[] paths=FileUtil.stat2Paths(null,defaultPath);
  assertEquals(1,paths.length);
  assertEquals(defaultPath,paths[0]);
  paths=FileUtil.stat2Paths(null,null);
  assertTrue(paths != null);
  assertEquals(1,paths.length);
  assertEquals(null,paths[0]);
  Path path1=new Path("file://foo");
  Path path2=new Path("file://moo");
  FileStatus[] fileStatuses=new FileStatus[]{new FileStatus(3,false,0,0,0,path1),new FileStatus(3,false,0,0,0,path2)};
  paths=FileUtil.stat2Paths(fileStatuses,defaultPath);
  assertEquals(2,paths.length);
  assertEquals(paths[0],path1);
  assertEquals(paths[1],path2);
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testFullyDeleteContents() throws IOException {
  setupDirs();
  boolean ret=FileUtil.fullyDeleteContents(del);
  Assert.assertTrue(ret);
  Assert.assertTrue(del.exists());
  Assert.assertEquals(0,del.listFiles().length);
  validateTmpDir();
}

APIUtilityVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testStat2Paths1(){
  assertNull(FileUtil.stat2Paths(null));
  FileStatus[] fileStatuses=new FileStatus[0];
  Path[] paths=FileUtil.stat2Paths(fileStatuses);
  assertEquals(0,paths.length);
  Path path1=new Path("file://foo");
  Path path2=new Path("file://moo");
  fileStatuses=new FileStatus[]{new FileStatus(3,false,0,0,0,path1),new FileStatus(3,false,0,0,0,path2)};
  paths=FileUtil.stat2Paths(fileStatuses);
  assertEquals(2,paths.length);
  assertEquals(paths[0],path1);
  assertEquals(paths[1],path2);
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testCopy5() throws IOException {
  setupDirs();
  URI uri=tmp.toURI();
  Configuration conf=new Configuration();
  FileSystem fs=FileSystem.newInstance(uri,conf);
  final String content="some-content";
  File srcFile=createFile(tmp,"src",content);
  Path srcPath=new Path(srcFile.toURI());
  final File dest=new File(del,"dest");
  boolean result=FileUtil.copy(fs,srcPath,dest,false,conf);
  assertTrue(result);
  assertTrue(dest.exists());
  assertEquals(content.getBytes().length + System.getProperty("line.separator").getBytes().length,dest.length());
  assertTrue(srcFile.exists());
  dest.delete();
  assertTrue(!dest.exists());
  result=FileUtil.copy(fs,srcPath,dest,true,conf);
  assertTrue(result);
  assertTrue(dest.exists());
  assertEquals(content.getBytes().length + System.getProperty("line.separator").getBytes().length,dest.length());
  assertTrue(!srcFile.exists());
  dest.delete();
  assertTrue(!dest.exists());
  srcPath=new Path(partitioned.toURI());
  result=FileUtil.copy(fs,srcPath,dest,true,conf);
  assertTrue(result);
  assertTrue(dest.exists() && dest.isDirectory());
  File[] files=dest.listFiles();
  assertTrue(files != null);
  assertEquals(2,files.length);
  for (  File f : files) {
    assertEquals(3 + System.getProperty("line.separator").getBytes().length,f.length());
  }
  assertTrue(!partitioned.exists());
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests if fullyDelete deletes
 * (a) dangling symlink to file properly
 * (b) dangling symlink to directory properly
 * @throws IOException
 */
@Test(timeout=30000) public void testFullyDeleteDanglingSymlinks() throws IOException {
  setupDirs();
  boolean ret=FileUtil.fullyDelete(tmp);
  Assert.assertTrue(ret);
  Assert.assertFalse(tmp.exists());
  File link=new File(del,LINK);
  Assert.assertEquals(5,del.list().length);
  ret=FileUtil.fullyDelete(link);
  Assert.assertTrue(ret);
  Assert.assertEquals(4,del.list().length);
  File linkDir=new File(del,"tmpDir");
  ret=FileUtil.fullyDelete(linkDir);
  Assert.assertTrue(ret);
  Assert.assertEquals(3,del.list().length);
}

APIUtilityVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that getDU is able to handle cycles caused due to symbolic links
 * and that directory sizes are not added to the final calculated size
 * @throws IOException
 */
@Test(timeout=30000) public void testGetDU() throws Exception {
  setupDirs();
  long du=FileUtil.getDU(TEST_DIR);
  final long expected=2 * (3 + System.getProperty("line.separator").length());
  Assert.assertEquals(expected,du);
  final File doesNotExist=new File(tmp,"QuickBrownFoxJumpsOverTheLazyDog");
  long duDoesNotExist=FileUtil.getDU(doesNotExist);
  assertEquals(0,duDoesNotExist);
  File notADirectory=new File(partitioned,"part-r-00000");
  long duNotADirectoryActual=FileUtil.getDU(notADirectory);
  long duNotADirectoryExpected=3 + System.getProperty("line.separator").length();
  assertEquals(duNotADirectoryExpected,duNotADirectoryActual);
  try {
    try {
      FileUtil.chmod(notADirectory.getAbsolutePath(),"0000");
    }
 catch (    InterruptedException ie) {
      assertNull(ie);
    }
    assertFalse(FileUtil.canRead(notADirectory));
    final long du3=FileUtil.getDU(partitioned);
    assertEquals(expected,du3);
    try {
      FileUtil.chmod(partitioned.getAbsolutePath(),"0000");
    }
 catch (    InterruptedException ie) {
      assertNull(ie);
    }
    assertFalse(FileUtil.canRead(partitioned));
    final long du4=FileUtil.getDU(partitioned);
    assertEquals(0,du4);
  }
  finally {
    FileUtil.chmod(partitioned.getAbsolutePath(),"0777",true);
  }
}

APIUtilityVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testListFiles() throws IOException {
  setupDirs();
  File[] files=FileUtil.listFiles(partitioned);
  Assert.assertEquals(2,files.length);
  File newDir=new File(tmp.getPath(),"test");
  newDir.mkdir();
  Assert.assertTrue("Failed to create test dir",newDir.exists());
  files=FileUtil.listFiles(newDir);
  Assert.assertEquals(0,files.length);
  newDir.delete();
  Assert.assertFalse("Failed to delete test dir",newDir.exists());
  try {
    files=FileUtil.listFiles(newDir);
    Assert.fail("IOException expected on listFiles() for non-existent dir " + newDir.toString());
  }
 catch (  IOException ioe) {
  }
}

APIUtilityVerifier BooleanVerifier 

@Test(timeout=30000) public void testCreateLocalTempFile() throws IOException {
  setupDirs();
  final File baseFile=new File(tmp,"base");
  File tmp1=FileUtil.createLocalTempFile(baseFile,"foo",false);
  File tmp2=FileUtil.createLocalTempFile(baseFile,"foo",true);
  assertFalse(tmp1.getAbsolutePath().equals(baseFile.getAbsolutePath()));
  assertFalse(tmp2.getAbsolutePath().equals(baseFile.getAbsolutePath()));
  assertTrue(tmp1.exists() && tmp2.exists());
  assertTrue(tmp1.canWrite() && tmp2.canWrite());
  assertTrue(tmp1.canRead() && tmp2.canRead());
  tmp1.delete();
  tmp2.delete();
  assertTrue(!tmp1.exists() && !tmp2.exists());
}

APIUtilityVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testCreateJarWithClassPath() throws Exception {
  Assert.assertFalse(tmp.exists());
  Assert.assertTrue(tmp.mkdirs());
  List wildcardMatches=Arrays.asList(new File(tmp,"wildcard1.jar"),new File(tmp,"wildcard2.jar"),new File(tmp,"wildcard3.JAR"),new File(tmp,"wildcard4.JAR"));
  for (  File wildcardMatch : wildcardMatches) {
    Assert.assertTrue("failure creating file: " + wildcardMatch,wildcardMatch.createNewFile());
  }
  Assert.assertTrue(new File(tmp,"text.txt").createNewFile());
  Assert.assertTrue(new File(tmp,"executable.exe").createNewFile());
  Assert.assertTrue(new File(tmp,"README").createNewFile());
  String wildcardPath=tmp.getCanonicalPath() + File.separator + "*";
  String nonExistentSubdir=tmp.getCanonicalPath() + Path.SEPARATOR + "subdir"+ Path.SEPARATOR;
  List classPaths=Arrays.asList("","cp1.jar","cp2.jar",wildcardPath,"cp3.jar",nonExistentSubdir);
  String inputClassPath=StringUtils.join(File.pathSeparator,classPaths);
  String classPathJar=FileUtil.createJarWithClassPath(inputClassPath,new Path(tmp.getCanonicalPath()),System.getenv());
  JarFile jarFile=null;
  try {
    jarFile=new JarFile(classPathJar);
    Manifest jarManifest=jarFile.getManifest();
    Assert.assertNotNull(jarManifest);
    Attributes mainAttributes=jarManifest.getMainAttributes();
    Assert.assertNotNull(mainAttributes);
    Assert.assertTrue(mainAttributes.containsKey(Attributes.Name.CLASS_PATH));
    String classPathAttr=mainAttributes.getValue(Attributes.Name.CLASS_PATH);
    Assert.assertNotNull(classPathAttr);
    List expectedClassPaths=new ArrayList();
    for (    String classPath : classPaths) {
      if (classPath.length() == 0) {
        continue;
      }
      if (wildcardPath.equals(classPath)) {
        for (        File wildcardMatch : wildcardMatches) {
          expectedClassPaths.add(wildcardMatch.toURI().toURL().toExternalForm());
        }
      }
 else {
        File fileCp=null;
        if (!new Path(classPath).isAbsolute()) {
          fileCp=new File(tmp,classPath);
        }
 else {
          fileCp=new File(classPath);
        }
        if (nonExistentSubdir.equals(classPath)) {
          expectedClassPaths.add(fileCp.toURI().toURL().toExternalForm() + Path.SEPARATOR);
        }
 else {
          expectedClassPaths.add(fileCp.toURI().toURL().toExternalForm());
        }
      }
    }
    List actualClassPaths=Arrays.asList(classPathAttr.split(" "));
    Collections.sort(expectedClassPaths);
    Collections.sort(actualClassPaths);
    Assert.assertEquals(expectedClassPaths,actualClassPaths);
  }
  finally {
    if (jarFile != null) {
      try {
        jarFile.close();
      }
 catch (      IOException e) {
        LOG.warn("exception closing jarFile: " + classPathJar,e);
      }
    }
  }
}

Class: org.apache.hadoop.fs.TestFsShellReturnCode

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testGetWithInvalidSourcePathShouldNotDisplayNullInConsole() throws Exception {
  Configuration conf=new Configuration();
  FsShell shell=new FsShell();
  shell.setConf(conf);
  final ByteArrayOutputStream bytes=new ByteArrayOutputStream();
  final PrintStream out=new PrintStream(bytes);
  final PrintStream oldErr=System.err;
  System.setErr(out);
  final String results;
  try {
    Path tdir=new Path(TEST_ROOT_DIR,"notNullCopy");
    fileSys.delete(tdir,true);
    fileSys.mkdirs(tdir);
    String[] args=new String[3];
    args[0]="-get";
    args[1]=new Path(tdir.toUri().getPath(),"/invalidSrc").toString();
    args[2]=new Path(tdir.toUri().getPath(),"/invalidDst").toString();
    assertTrue("file exists",!fileSys.exists(new Path(args[1])));
    assertTrue("file exists",!fileSys.exists(new Path(args[2])));
    int run=shell.run(args);
    results=bytes.toString();
    assertEquals("Return code should be 1",1,run);
    assertTrue(" Null is coming when source path is invalid. ",!results.contains("get: null"));
    assertTrue(" Not displaying the intended message ",results.contains("get: `" + args[1] + "': No such file or directory"));
  }
  finally {
    IOUtils.closeStream(out);
    System.setErr(oldErr);
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testRmWithNonexistentGlob() throws Exception {
  Configuration conf=new Configuration();
  FsShell shell=new FsShell();
  shell.setConf(conf);
  final ByteArrayOutputStream bytes=new ByteArrayOutputStream();
  final PrintStream err=new PrintStream(bytes);
  final PrintStream oldErr=System.err;
  System.setErr(err);
  final String results;
  try {
    int exit=shell.run(new String[]{"-rm","nomatch*"});
    assertEquals(1,exit);
    results=bytes.toString();
    assertTrue(results.contains("rm: `nomatch*': No such file or directory"));
  }
  finally {
    IOUtils.closeStream(err);
    System.setErr(oldErr);
  }
}

Class: org.apache.hadoop.fs.TestGlobPaths

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void pTestCurlyBracket() throws IOException {
  Path[] matchedPath;
  String[] files;
  try {
    files=new String[]{USER_DIR + "/a.abcxx",USER_DIR + "/a.abxy",USER_DIR + "/a.hlp",USER_DIR + "/a.jhyy"};
    matchedPath=prepareTesting(USER_DIR + "/a.{abc,jh}??",files);
    assertEquals(matchedPath.length,2);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[3]);
  }
  finally {
    cleanupDFS();
  }
  try {
    files=new String[]{USER_DIR + "/a.abcxx",USER_DIR + "/a.abdxy",USER_DIR + "/a.hlp",USER_DIR + "/a.jhyy"};
    matchedPath=prepareTesting(USER_DIR + "/a.{ab{c,d},jh}??",files);
    assertEquals(matchedPath.length,3);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[1]);
    assertEquals(matchedPath[2],path[3]);
  }
  finally {
    cleanupDFS();
  }
  try {
    files=new String[]{USER_DIR + "/a/b",USER_DIR + "/a/d",USER_DIR + "/c/b",USER_DIR + "/c/d"};
    matchedPath=prepareTesting(USER_DIR + "/{a/b,c/d}",files);
    assertEquals(matchedPath.length,2);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[3]);
  }
  finally {
    cleanupDFS();
  }
  try {
    files=new String[]{"/a/b","/a/d","/c/b","/c/d"};
    matchedPath=prepareTesting("{/a/b,/c/d}",files);
    assertEquals(matchedPath.length,2);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[3]);
  }
  finally {
    cleanupDFS();
  }
  try {
    files=new String[]{USER_DIR + "/}bc",USER_DIR + "/}c"};
    matchedPath=prepareTesting(USER_DIR + "/}{a,b}c",files);
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[0]);
    matchedPath=prepareTesting(USER_DIR + "/}{b}c",files);
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[0]);
    matchedPath=prepareTesting(USER_DIR + "/}{}bc",files);
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[0]);
    matchedPath=prepareTesting(USER_DIR + "/}{,}bc",files);
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[0]);
    matchedPath=prepareTesting(USER_DIR + "/}{b,}c",files);
    assertEquals(matchedPath.length,2);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[1]);
    matchedPath=prepareTesting(USER_DIR + "/}{,b}c",files);
    assertEquals(matchedPath.length,2);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[1]);
    matchedPath=prepareTesting(USER_DIR + "/}{ac,?}",files);
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[1]);
    boolean hasException=false;
    try {
      prepareTesting(USER_DIR + "}{bc",files);
    }
 catch (    IOException e) {
      assertTrue(e.getMessage().startsWith("Illegal file pattern:"));
      hasException=true;
    }
    assertTrue(hasException);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestClosure2() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/a.",USER_DIR + "/a.txt",USER_DIR + "/a.old.java",USER_DIR + "/.java"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/a.*",files);
    assertEquals(matchedPath.length,3);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[2]);
    assertEquals(matchedPath[2],path[1]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier AssumptionSetter EqualityVerifier HybridVerifier 

@Test public void pTestEscape() throws IOException {
  org.junit.Assume.assumeTrue(!Path.WINDOWS);
  try {
    String[] files=new String[]{USER_DIR + "/ab\\[c.d"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/ab\\[c.d",files);
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[0]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestClosure1() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/a",USER_DIR + "/abc",USER_DIR + "/abc.p",USER_DIR + "/bacd"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/a*",files);
    assertEquals(matchedPath.length,3);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[1]);
    assertEquals(matchedPath[2],path[2]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestClosure4() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/dir1/file1",USER_DIR + "/dir2/file2",USER_DIR + "/dir3/file1"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/*/file1",files);
    assertEquals(matchedPath.length,2);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[2]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestJavaRegexSpecialChars() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/($.|+)bc",USER_DIR + "/abc"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/($.|+)*",files);
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[0]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestClosure3() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/a.txt.x",USER_DIR + "/ax",USER_DIR + "/ab37x",USER_DIR + "/bacd"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/a*x",files);
    assertEquals(matchedPath.length,3);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[2]);
    assertEquals(matchedPath[2],path[1]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestLiteral() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/a2c",USER_DIR + "/abc.d"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/abc.d",files);
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[1]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestSetExcl() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/a.d",USER_DIR + "/a.e",USER_DIR + "/a.0",USER_DIR + "/a.h"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/a.[^a-cg-z0-9]",files);
    assertEquals(matchedPath.length,2);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[1]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestCombination() throws IOException {
  try {
    String[] files=new String[]{"/user/aa/a.c","/user/bb/a.cpp","/user1/cc/b.hlp","/user/dd/a.hxy"};
    Path[] matchedPath=prepareTesting("/use?/*/a.[ch]{lp,xy}",files);
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[3]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void testPathFilter() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/a",USER_DIR + "/a/b"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/*/*",files,new RegexPathFilter("^.*" + Pattern.quote(USER_DIR) + "/a/b"));
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[1]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestRange() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/a.d",USER_DIR + "/a.e",USER_DIR + "/a.f",USER_DIR + "/a.h"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/a.[d-fm]",files);
    assertEquals(matchedPath.length,3);
    assertEquals(matchedPath[0],path[0]);
    assertEquals(matchedPath[1],path[1]);
    assertEquals(matchedPath[2],path[2]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestAny() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/abc",USER_DIR + "/a2c",USER_DIR + "/a.c",USER_DIR + "/abcd"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/a?c",files);
    assertEquals(matchedPath.length,3);
    assertEquals(matchedPath[0],path[2]);
    assertEquals(matchedPath[1],path[1]);
    assertEquals(matchedPath[2],path[0]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void testPathFilterWithFixedLastComponent() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/a",USER_DIR + "/a/b",USER_DIR + "/c",USER_DIR + "/c/b"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/*/b",files,new RegexPathFilter("^.*" + Pattern.quote(USER_DIR) + "/a/b"));
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[1]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestClosure5() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/dir1/file1",USER_DIR + "/file1"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/*/file1",files);
    assertEquals(matchedPath.length,1);
    assertEquals(matchedPath[0],path[0]);
  }
  finally {
    cleanupDFS();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void pTestSet() throws IOException {
  try {
    String[] files=new String[]{USER_DIR + "/a.c",USER_DIR + "/a.cpp",USER_DIR + "/a.hlp",USER_DIR + "/a.hxy"};
    Path[] matchedPath=prepareTesting(USER_DIR + "/a.[ch]??",files);
    assertEquals(matchedPath.length,3);
    assertEquals(matchedPath[0],path[1]);
    assertEquals(matchedPath[1],path[2]);
    assertEquals(matchedPath[2],path[3]);
  }
  finally {
    cleanupDFS();
  }
}

Class: org.apache.hadoop.fs.TestHarFileSystemBasics

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testMakeQualifiedPath() throws Exception {
  String harPathWithUserinfo="har://file-user:passwd@localhost:80" + harPath.toUri().getPath().toString();
  Path path=new Path(harPathWithUserinfo);
  Path qualifiedPath=path.getFileSystem(conf).makeQualified(path);
  assertTrue(String.format("The qualified path (%s) did not match the expected path (%s).",qualifiedPath.toString(),harPathWithUserinfo),qualifiedPath.toString().equals(harPathWithUserinfo));
}

Class: org.apache.hadoop.fs.TestHardLink

APIUtilityVerifier EqualityVerifier 

/** 
 * Test createHardLinkMult() with empty list of files.
 * We use an extended version of the method call, that
 * returns the number of System exec calls made, which should
 * be zero in this case.
 */
@Test public void testCreateHardLinkMultEmptyList() throws IOException {
  String[] emptyList={};
  int callCount=createHardLinkMult(src,emptyList,tgt_mult,getMaxAllowedCmdArgLength());
  assertEquals(0,callCount);
  validateSetup();
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test createHardLinkMult(), again, this time with the "too long list" 
 * case where the total size of the command line arguments exceed the 
 * allowed maximum.  In this case, the list should be automatically 
 * broken up into chunks, each chunk no larger than the max allowed.
 * We use an extended version of the method call, specifying the
 * size limit explicitly, to simulate the "too long" list with a 
 * relatively short list.
 */
@Test public void testCreateHardLinkMultOversizeAndEmpty() throws IOException {
  String name1="x11111111";
  String name2="x22222222";
  String name3="x33333333";
  File x1_long=new File(src,name1);
  File x2_long=new File(src,name2);
  File x3_long=new File(src,name3);
  x1.renameTo(x1_long);
  x2.renameTo(x2_long);
  x3.renameTo(x3_long);
  assertTrue(x1_long.exists());
  assertTrue(x2_long.exists());
  assertTrue(x3_long.exists());
  assertFalse(x1.exists());
  assertFalse(x2.exists());
  assertFalse(x3.exists());
  int callCount;
  String[] emptyList={};
  String[] fileNames=src.list();
  int overhead=getLinkMultArgLength(src,emptyList,tgt_mult);
  int maxLength=overhead + (int)(2.5 * (float)(1 + name1.length()));
  callCount=createHardLinkMult(src,fileNames,tgt_mult,maxLength);
  assertEquals(2,callCount);
  String[] tgt_multNames=tgt_mult.list();
  Arrays.sort(fileNames);
  Arrays.sort(tgt_multNames);
  assertArrayEquals(fileNames,tgt_multNames);
  FileUtil.fullyDelete(tgt_mult);
  assertFalse(tgt_mult.exists());
  tgt_mult.mkdirs();
  assertTrue(tgt_mult.exists() && tgt_mult.list().length == 0);
  maxLength=overhead + (int)(0.5 * (float)(1 + name1.length()));
  callCount=createHardLinkMult(src,fileNames,tgt_mult,maxLength);
  assertEquals(3,callCount);
  tgt_multNames=tgt_mult.list();
  Arrays.sort(fileNames);
  Arrays.sort(tgt_multNames);
  assertArrayEquals(fileNames,tgt_multNames);
}

Class: org.apache.hadoop.fs.TestLocalDirAllocator

APIUtilityVerifier BooleanVerifier 

/** 
 * Two buffer dirs. The first dir exists & is on a read-only disk;
 * The second dir exists & is RW
 * @throws Exception
 */
@Test(timeout=30000) public void testROBufferDirAndRWBufferDir() throws Exception {
  if (isWindows)   return;
  String dir1=buildBufferDir(ROOT,1);
  String dir2=buildBufferDir(ROOT,2);
  try {
    conf.set(CONTEXT,dir1 + "," + dir2);
    assertTrue(localFs.mkdirs(new Path(dir2)));
    BUFFER_ROOT.setReadOnly();
    validateTempDirCreation(dir2);
    validateTempDirCreation(dir2);
  }
  finally {
    Shell.execCommand(Shell.getSetPermissionCommand("u+w",false,BUFFER_DIR_ROOT));
    rmBufferDirs();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=30000) public void testCreateManyFiles() throws Exception {
  if (isWindows)   return;
  String dir5=buildBufferDir(ROOT,5);
  String dir6=buildBufferDir(ROOT,6);
  try {
    conf.set(CONTEXT,dir5 + "," + dir6);
    assertTrue(localFs.mkdirs(new Path(dir5)));
    assertTrue(localFs.mkdirs(new Path(dir6)));
    int inDir5=0, inDir6=0;
    for (int i=0; i < TRIALS; ++i) {
      File result=createTempFile();
      if (result.getPath().startsWith(new Path(dir5,FILENAME).toUri().getPath())) {
        inDir5++;
      }
 else       if (result.getPath().startsWith(new Path(dir6,FILENAME).toUri().getPath())) {
        inDir6++;
      }
      result.delete();
    }
    assertTrue(inDir5 + inDir6 == TRIALS);
  }
  finally {
    rmBufferDirs();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier AssumptionSetter EqualityVerifier HybridVerifier 

/** 
 * Test that {@link LocalDirAllocator#getAllLocalPathsToRead(String,Configuration)} 
 * returns correct filenames and "file" schema.
 * @throws IOException
 */
@Test(timeout=30000) public void testGetAllLocalPathsToRead() throws IOException {
  assumeTrue(!isWindows);
  String dir0=buildBufferDir(ROOT,0);
  String dir1=buildBufferDir(ROOT,1);
  try {
    conf.set(CONTEXT,dir0 + "," + dir1);
    assertTrue(localFs.mkdirs(new Path(dir0)));
    assertTrue(localFs.mkdirs(new Path(dir1)));
    localFs.create(new Path(dir0 + Path.SEPARATOR + FILENAME));
    localFs.create(new Path(dir1 + Path.SEPARATOR + FILENAME));
    final Iterable pathIterable=dirAllocator.getAllLocalPathsToRead(FILENAME,conf);
    int count=0;
    for (    final Path p : pathIterable) {
      count++;
      assertEquals(FILENAME,p.getName());
      assertEquals("file",p.getFileSystem(conf).getUri().getScheme());
    }
    assertEquals(2,count);
    try {
      Path p=pathIterable.iterator().next();
      assertFalse("NoSuchElementException must be thrown, but returned [" + p + "] instead.",true);
    }
 catch (    NoSuchElementException nsee) {
    }
    final Iterable pathIterable2=dirAllocator.getAllLocalPathsToRead(FILENAME,conf);
    final Iterator it=pathIterable2.iterator();
    try {
      it.remove();
      assertFalse(true);
    }
 catch (    UnsupportedOperationException uoe) {
    }
  }
  finally {
    Shell.execCommand(new String[]{"chmod","u+w",BUFFER_DIR_ROOT});
    rmBufferDirs();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier AssumptionSetter HybridVerifier 

/** 
 * Test no side effect files are left over. After creating a temp
 * temp file, remove both the temp file and its parent. Verify that
 * no files or directories are left over as can happen when File objects
 * are mistakenly created from fully qualified path strings.
 * @throws IOException
 */
@Test(timeout=30000) public void testNoSideEffects() throws IOException {
  assumeTrue(!isWindows);
  String dir=buildBufferDir(ROOT,0);
  try {
    conf.set(CONTEXT,dir);
    File result=dirAllocator.createTmpFileForWrite(FILENAME,-1,conf);
    assertTrue(result.delete());
    assertTrue(result.getParentFile().delete());
    assertFalse(new File(dir).exists());
  }
  finally {
    Shell.execCommand(Shell.getSetPermissionCommand("u+w",false,BUFFER_DIR_ROOT));
    rmBufferDirs();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier AssumptionSetter EqualityVerifier HybridVerifier 

/** 
 * Test getLocalPathToRead() returns correct filename and "file" schema.
 * @throws IOException
 */
@Test(timeout=30000) public void testGetLocalPathToRead() throws IOException {
  assumeTrue(!isWindows);
  String dir=buildBufferDir(ROOT,0);
  try {
    conf.set(CONTEXT,dir);
    assertTrue(localFs.mkdirs(new Path(dir)));
    File f1=dirAllocator.createTmpFileForWrite(FILENAME,SMALL_FILE_SIZE,conf);
    Path p1=dirAllocator.getLocalPathToRead(f1.getName(),conf);
    assertEquals(f1.getName(),p1.getName());
    assertEquals("file",p1.getFileSystem(conf).getUri().getScheme());
  }
  finally {
    Shell.execCommand(Shell.getSetPermissionCommand("u+w",false,BUFFER_DIR_ROOT));
    rmBufferDirs();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Two buffer dirs. Both exists and on a R/W disk.
 * Later disk1 becomes read-only.
 * @throws Exception
 */
@Test(timeout=30000) public void testRWBufferDirBecomesRO() throws Exception {
  if (isWindows)   return;
  String dir3=buildBufferDir(ROOT,3);
  String dir4=buildBufferDir(ROOT,4);
  try {
    conf.set(CONTEXT,dir3 + "," + dir4);
    assertTrue(localFs.mkdirs(new Path(dir3)));
    assertTrue(localFs.mkdirs(new Path(dir4)));
    createTempFile(SMALL_FILE_SIZE);
    int nextDirIdx=(dirAllocator.getCurrentDirectoryIndex() == 0) ? 3 : 4;
    validateTempDirCreation(buildBufferDir(ROOT,nextDirIdx));
    new File(new Path(dir4).toUri().getPath()).setReadOnly();
    validateTempDirCreation(dir3);
    validateTempDirCreation(dir3);
  }
  finally {
    rmBufferDirs();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Two buffer dirs. The first dir does not exist & is on a read-only disk;
 * The second dir exists & is RW
 * getLocalPathForWrite with checkAccess set to false should create a parent
 * directory. With checkAccess true, the directory should not be created.
 * @throws Exception
 */
@Test(timeout=30000) public void testLocalPathForWriteDirCreation() throws IOException {
  String dir0=buildBufferDir(ROOT,0);
  String dir1=buildBufferDir(ROOT,1);
  try {
    conf.set(CONTEXT,dir0 + "," + dir1);
    assertTrue(localFs.mkdirs(new Path(dir1)));
    BUFFER_ROOT.setReadOnly();
    Path p1=dirAllocator.getLocalPathForWrite("p1/x",SMALL_FILE_SIZE,conf);
    assertTrue(localFs.getFileStatus(p1.getParent()).isDirectory());
    Path p2=dirAllocator.getLocalPathForWrite("p2/x",SMALL_FILE_SIZE,conf,false);
    try {
      localFs.getFileStatus(p2.getParent());
    }
 catch (    Exception e) {
      assertEquals(e.getClass(),FileNotFoundException.class);
    }
  }
  finally {
    Shell.execCommand(Shell.getSetPermissionCommand("u+w",false,BUFFER_DIR_ROOT));
    rmBufferDirs();
  }
}

APIUtilityVerifier BooleanVerifier 

/** 
 * Two buffer dirs. The first dir does not exist & is on a read-only disk;
 * The second dir exists & is RW
 * @throws Exception
 */
@Test(timeout=30000) public void test0() throws Exception {
  if (isWindows)   return;
  String dir0=buildBufferDir(ROOT,0);
  String dir1=buildBufferDir(ROOT,1);
  try {
    conf.set(CONTEXT,dir0 + "," + dir1);
    assertTrue(localFs.mkdirs(new Path(dir1)));
    BUFFER_ROOT.setReadOnly();
    validateTempDirCreation(dir1);
    validateTempDirCreation(dir1);
  }
  finally {
    Shell.execCommand(Shell.getSetPermissionCommand("u+w",false,BUFFER_DIR_ROOT));
    rmBufferDirs();
  }
}

Class: org.apache.hadoop.fs.TestLocalFileSystem

APIUtilityVerifier BranchVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=10000) public void testReportChecksumFailure() throws IOException {
  base.mkdirs();
  assertTrue(base.exists() && base.isDirectory());
  final File dir1=new File(base,"dir1");
  final File dir2=new File(dir1,"dir2");
  dir2.mkdirs();
  assertTrue(dir2.exists() && FileUtil.canWrite(dir2));
  final String dataFileName="corruptedData";
  final Path dataPath=new Path(new File(dir2,dataFileName).toURI());
  final Path checksumPath=fileSys.getChecksumFile(dataPath);
  final FSDataOutputStream fsdos=fileSys.create(dataPath);
  try {
    fsdos.writeUTF("foo");
  }
  finally {
    fsdos.close();
  }
  assertTrue(fileSys.pathToFile(dataPath).exists());
  final long dataFileLength=fileSys.getFileStatus(dataPath).getLen();
  assertTrue(dataFileLength > 0);
  assertTrue(fileSys.pathToFile(checksumPath).exists());
  final long checksumFileLength=fileSys.getFileStatus(checksumPath).getLen();
  assertTrue(checksumFileLength > 0);
  FileUtil.setWritable(base,false);
  FSDataInputStream dataFsdis=fileSys.open(dataPath);
  FSDataInputStream checksumFsdis=fileSys.open(checksumPath);
  boolean retryIsNecessary=fileSys.reportChecksumFailure(dataPath,dataFsdis,0,checksumFsdis,0);
  assertTrue(!retryIsNecessary);
  assertTrue(!fileSys.pathToFile(dataPath).exists());
  assertTrue(!fileSys.pathToFile(checksumPath).exists());
  File[] dir1files=dir1.listFiles(new FileFilter(){
    @Override public boolean accept(    File pathname){
      return pathname != null && !pathname.getName().equals("dir2");
    }
  }
);
  assertTrue(dir1files != null);
  assertTrue(dir1files.length == 1);
  File badFilesDir=dir1files[0];
  File[] badFiles=badFilesDir.listFiles();
  assertTrue(badFiles != null);
  assertTrue(badFiles.length == 2);
  boolean dataFileFound=false;
  boolean checksumFileFound=false;
  for (  File badFile : badFiles) {
    if (badFile.getName().startsWith(dataFileName)) {
      assertTrue(dataFileLength == badFile.length());
      dataFileFound=true;
    }
 else     if (badFile.getName().contains(dataFileName + ".crc")) {
      assertTrue(checksumFileLength == badFile.length());
      checksumFileFound=true;
    }
  }
  assertTrue(dataFileFound);
  assertTrue(checksumFileFound);
}

APIUtilityVerifier InternalCallVerifier AssumptionSetter EqualityVerifier HybridVerifier 

@Test public void testListStatusReturnConsistentPathOnWindows() throws IOException {
  assumeTrue(Shell.WINDOWS);
  String dirNoDriveSpec=TEST_ROOT_DIR;
  if (dirNoDriveSpec.charAt(1) == ':')   dirNoDriveSpec=dirNoDriveSpec.substring(2);
  File file=new File(dirNoDriveSpec,"foo");
  file.mkdirs();
  FileStatus[] stats=fileSys.listStatus(new Path(dirNoDriveSpec));
  assertEquals("Unexpected number of stats",1,stats.length);
  assertEquals("Bad path from stat",new Path(file.getPath()).toUri().getPath(),stats[0].getPath().toUri().getPath());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test the capability of setting the working directory.
 */
@Test(timeout=10000) public void testWorkingDirectory() throws IOException {
  Path origDir=fileSys.getWorkingDirectory();
  Path subdir=new Path(TEST_ROOT_DIR,"new");
  try {
    assertTrue(!fileSys.exists(subdir));
    assertTrue(fileSys.mkdirs(subdir));
    assertTrue(fileSys.isDirectory(subdir));
    fileSys.setWorkingDirectory(subdir);
    Path dir1=new Path("dir1");
    assertTrue(fileSys.mkdirs(dir1));
    assertTrue(fileSys.isDirectory(dir1));
    fileSys.delete(dir1,true);
    assertTrue(!fileSys.exists(dir1));
    Path file1=new Path("file1");
    Path file2=new Path("sub/file2");
    String contents=writeFile(fileSys,file1,1);
    fileSys.copyFromLocalFile(file1,file2);
    assertTrue(fileSys.exists(file1));
    assertTrue(fileSys.isFile(file1));
    cleanupFile(fileSys,file2);
    fileSys.copyToLocalFile(file1,file2);
    cleanupFile(fileSys,file2);
    fileSys.rename(file1,file2);
    assertTrue(!fileSys.exists(file1));
    assertTrue(fileSys.exists(file2));
    fileSys.rename(file2,file1);
    InputStream stm=fileSys.open(file1);
    byte[] buffer=new byte[3];
    int bytesRead=stm.read(buffer,0,3);
    assertEquals(contents,new String(buffer,0,bytesRead));
    stm.close();
  }
  finally {
    fileSys.setWorkingDirectory(origDir);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testStripFragmentFromPath() throws Exception {
  FileSystem fs=FileSystem.getLocal(new Configuration());
  Path pathQualified=TEST_PATH.makeQualified(fs.getUri(),fs.getWorkingDirectory());
  Path pathWithFragment=new Path(new URI(pathQualified.toString() + "#glacier"));
  FileSystemTestHelper.createFile(fs,pathWithFragment);
  Path resolved=fs.resolvePath(pathWithFragment);
  assertEquals("resolvePath did not strip fragment from Path",pathQualified,resolved);
}

Class: org.apache.hadoop.fs.TestStat

APIUtilityVerifier InternalCallVerifier BooleanVerifier AssumptionSetter HybridVerifier 

@Test(timeout=10000) public void testStat() throws Exception {
  Assume.assumeTrue(Stat.isAvailable());
  FileSystem fs=FileSystem.getLocal(new Configuration());
  Path testDir=new Path(getTestRootPath(fs),"teststat");
  fs.mkdirs(testDir);
  Path sub1=new Path(testDir,"sub1");
  Path sub2=new Path(testDir,"sub2");
  fs.mkdirs(sub1);
  fs.createSymlink(sub1,sub2,false);
  FileStatus stat1=new Stat(sub1,4096l,false,fs).getFileStatus();
  FileStatus stat2=new Stat(sub2,0,false,fs).getFileStatus();
  assertTrue(stat1.isDirectory());
  assertFalse(stat2.isDirectory());
  fs.delete(testDir,true);
}

Class: org.apache.hadoop.fs.TestSymlinkHdfs

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=10000) public void testSetPermissionAffectsTarget() throws IOException {
  Path file=new Path(testBaseDir1(),"file");
  Path dir=new Path(testBaseDir2());
  Path linkToFile=new Path(testBaseDir1(),"linkToFile");
  Path linkToDir=new Path(testBaseDir1(),"linkToDir");
  createAndWriteFile(file);
  wrapper.createSymlink(file,linkToFile,false);
  wrapper.createSymlink(dir,linkToDir,false);
  FsPermission perms=wrapper.getFileLinkStatus(linkToFile).getPermission();
  wrapper.setPermission(linkToFile,new FsPermission((short)0664));
  wrapper.setOwner(linkToFile,"user","group");
  assertEquals(perms,wrapper.getFileLinkStatus(linkToFile).getPermission());
  FileStatus stat=wrapper.getFileStatus(file);
  assertEquals(0664,stat.getPermission().toShort());
  assertEquals("user",stat.getOwner());
  assertEquals("group",stat.getGroup());
  assertEquals(stat.getPermission(),wrapper.getFileStatus(linkToFile).getPermission());
  perms=wrapper.getFileLinkStatus(linkToDir).getPermission();
  wrapper.setPermission(linkToDir,new FsPermission((short)0664));
  wrapper.setOwner(linkToDir,"user","group");
  assertEquals(perms,wrapper.getFileLinkStatus(linkToDir).getPermission());
  stat=wrapper.getFileStatus(dir);
  assertEquals(0664,stat.getPermission().toShort());
  assertEquals("user",stat.getOwner());
  assertEquals("group",stat.getGroup());
  assertEquals(stat.getPermission(),wrapper.getFileStatus(linkToDir).getPermission());
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

@Test(timeout=10000) public void testCreateLinkMaxPathLink() throws IOException {
  Path dir=new Path(testBaseDir1());
  Path file=new Path(testBaseDir1(),"file");
  final int maxPathLen=HdfsConstants.MAX_PATH_LENGTH;
  final int dirLen=dir.toString().length() + 1;
  int len=maxPathLen - dirLen;
  StringBuilder sb=new StringBuilder("");
  for (int i=0; i < (len / 10); i++) {
    sb.append("0123456789");
  }
  for (int i=0; i < (len % 10); i++) {
    sb.append("x");
  }
  Path link=new Path(sb.toString());
  assertEquals(maxPathLen,dirLen + link.toString().length());
  createAndWriteFile(file);
  wrapper.setWorkingDirectory(dir);
  wrapper.createSymlink(file,link,false);
  readFile(link);
  link=new Path(sb.toString() + "x");
  try {
    wrapper.createSymlink(file,link,false);
    fail("Path name should be too long");
  }
 catch (  IOException x) {
  }
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=10000) public void testLinkOwner() throws IOException {
  Path file=new Path(testBaseDir1(),"file");
  Path link=new Path(testBaseDir1(),"symlinkToFile");
  createAndWriteFile(file);
  wrapper.createSymlink(file,link,false);
  FileStatus statFile=wrapper.getFileStatus(file);
  FileStatus statLink=wrapper.getFileStatus(link);
  assertEquals(statLink.getOwner(),statFile.getOwner());
}

Class: org.apache.hadoop.fs.TestSymlinkHdfsFileSystem

APIUtilityVerifier BooleanVerifier 

@Test(timeout=10000) public void testRecoverLease() throws IOException {
  Path dir=new Path(testBaseDir1());
  Path file=new Path(testBaseDir1(),"file");
  Path link=new Path(testBaseDir1(),"link");
  wrapper.setWorkingDirectory(dir);
  createAndWriteFile(file);
  wrapper.createSymlink(file,link,false);
  boolean closed=dfs.recoverLease(link);
  assertTrue("Expected recoverLease to return true",closed);
}

APIUtilityVerifier BooleanVerifier 

@Test(timeout=10000) public void testIsFileClosed() throws IOException {
  Path dir=new Path(testBaseDir1());
  Path file=new Path(testBaseDir1(),"file");
  Path link=new Path(testBaseDir1(),"link");
  wrapper.setWorkingDirectory(dir);
  createAndWriteFile(file);
  wrapper.createSymlink(file,link,false);
  boolean closed=dfs.isFileClosed(link);
  assertTrue("Expected isFileClosed to return true",closed);
}

Class: org.apache.hadoop.fs.TestSymlinkLocalFS

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier AssumptionSetter EqualityVerifier HybridVerifier 

@Test(timeout=1000) public void testDanglingLink() throws IOException {
  assumeTrue(!Path.WINDOWS);
  Path fileAbs=new Path(testBaseDir1() + "/file");
  Path fileQual=new Path(testURI().toString(),fileAbs);
  Path link=new Path(testBaseDir1() + "/linkToFile");
  Path linkQual=new Path(testURI().toString(),link.toString());
  wrapper.createSymlink(fileAbs,link,false);
  FileUtil.fullyDelete(new File(link.toUri().getPath()));
  wrapper.createSymlink(fileAbs,link,false);
  try {
    wrapper.getFileStatus(link);
    fail("Got FileStatus for dangling link");
  }
 catch (  FileNotFoundException f) {
  }
  UserGroupInformation user=UserGroupInformation.getCurrentUser();
  FileStatus fsd=wrapper.getFileLinkStatus(link);
  assertEquals(fileQual,fsd.getSymlink());
  assertTrue(fsd.isSymlink());
  assertFalse(fsd.isDirectory());
  assertEquals(user.getUserName(),fsd.getOwner());
  assertEquals(user.getGroupNames()[0],fsd.getGroup());
  assertEquals(linkQual,fsd.getPath());
  try {
    readFile(link);
    fail("Got FileStatus for dangling link");
  }
 catch (  FileNotFoundException f) {
  }
  createAndWriteFile(fileAbs);
  wrapper.getFileStatus(link);
}

Class: org.apache.hadoop.fs.TestUrlStreamHandler

APIUtilityVerifier IterativeVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test opening and reading from an InputStream through a hdfs:// URL.
 * 

 * First generate a file with some content through the FileSystem API, then
 * try to open and read the file through the URL stream API.
 * @throws IOException
 */
@Test public void testDfsUrls() throws IOException {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  FileSystem fs=cluster.getFileSystem();
  FsUrlStreamHandlerFactory factory=new org.apache.hadoop.fs.FsUrlStreamHandlerFactory();
  java.net.URL.setURLStreamHandlerFactory(factory);
  Path filePath=new Path("/thefile");
  try {
    byte[] fileContent=new byte[1024];
    for (int i=0; i < fileContent.length; ++i)     fileContent[i]=(byte)i;
    OutputStream os=fs.create(filePath);
    os.write(fileContent);
    os.close();
    URI uri=fs.getUri();
    URL fileURL=new URL(uri.getScheme(),uri.getHost(),uri.getPort(),filePath.toString());
    InputStream is=fileURL.openStream();
    assertNotNull(is);
    byte[] bytes=new byte[4096];
    assertEquals(1024,is.read(bytes));
    is.close();
    for (int i=0; i < fileContent.length; ++i)     assertEquals(fileContent[i],bytes[i]);
    fs.delete(filePath,false);
  }
  finally {
    fs.close();
    cluster.shutdown();
  }
}

APIUtilityVerifier IterativeVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test opening and reading from an InputStream through a file:// URL.
 * @throws IOException
 * @throws URISyntaxException
 */
@Test public void testFileUrls() throws IOException, URISyntaxException {
  Configuration conf=new HdfsConfiguration();
  if (!TEST_ROOT_DIR.exists()) {
    if (!TEST_ROOT_DIR.mkdirs())     throw new IOException("Cannot create temporary directory: " + TEST_ROOT_DIR);
  }
  File tmpFile=new File(TEST_ROOT_DIR,"thefile");
  URI uri=tmpFile.toURI();
  FileSystem fs=FileSystem.get(uri,conf);
  try {
    byte[] fileContent=new byte[1024];
    for (int i=0; i < fileContent.length; ++i)     fileContent[i]=(byte)i;
    OutputStream os=fs.create(new Path(uri.getPath()));
    os.write(fileContent);
    os.close();
    URL fileURL=uri.toURL();
    InputStream is=fileURL.openStream();
    assertNotNull(is);
    byte[] bytes=new byte[4096];
    assertEquals(1024,is.read(bytes));
    is.close();
    for (int i=0; i < fileContent.length; ++i)     assertEquals(fileContent[i],bytes[i]);
    fs.delete(new Path(uri.getPath()),false);
  }
  finally {
    fs.close();
  }
}

Class: org.apache.hadoop.fs.azure.NativeAzureFileSystemBaseTest

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testStatistics() throws Exception {
  FileSystem.clearStatistics();
  FileSystem.Statistics stats=FileSystem.getStatistics("wasb",NativeAzureFileSystem.class);
  assertEquals(0,stats.getBytesRead());
  assertEquals(0,stats.getBytesWritten());
  Path newFile=new Path("testStats");
  writeString(newFile,"12345678");
  assertEquals(8,stats.getBytesWritten());
  assertEquals(0,stats.getBytesRead());
  String readBack=readString(newFile);
  assertEquals("12345678",readBack);
  assertEquals(8,stats.getBytesRead());
  assertEquals(8,stats.getBytesWritten());
  assertTrue(fs.delete(newFile,true));
  assertEquals(8,stats.getBytesRead());
  assertEquals(8,stats.getBytesWritten());
}

Class: org.apache.hadoop.fs.azure.TestAzureConcurrentOutOfBandIo

APIUtilityVerifier IterativeVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

@Test public void testReadOOBWrites() throws Exception {
  byte[] dataBlockWrite=new byte[UPLOAD_BLOCK_SIZE];
  byte[] dataBlockRead=new byte[UPLOAD_BLOCK_SIZE];
  DataOutputStream outputStream=testAccount.getStore().storefile("WASB_String.txt",new PermissionStatus("","",FsPermission.getDefault()));
  Arrays.fill(dataBlockWrite,(byte)255);
  for (int i=0; i < NUMBER_OF_BLOCKS; i++) {
    outputStream.write(dataBlockWrite);
  }
  outputStream.flush();
  outputStream.close();
  DataBlockWriter writeBlockTask=new DataBlockWriter(testAccount,"WASB_String.txt");
  writeBlockTask.startWriting();
  int count=0;
  DataInputStream inputStream=null;
  for (int i=0; i < 5; i++) {
    try {
      inputStream=testAccount.getStore().retrieve("WASB_String.txt",0);
      count=0;
      int c=0;
      while (c >= 0) {
        c=inputStream.read(dataBlockRead,0,UPLOAD_BLOCK_SIZE);
        if (c < 0) {
          break;
        }
        count+=c;
      }
    }
 catch (    IOException e) {
      System.out.println(e.getCause().toString());
      e.printStackTrace();
      fail();
    }
    if (null != inputStream) {
      inputStream.close();
    }
  }
  writeBlockTask.stopWriting();
  assertEquals(NUMBER_OF_BLOCKS * UPLOAD_BLOCK_SIZE,count);
}

Class: org.apache.hadoop.fs.azure.TestBlobMetadata

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests that WASB understands the old-style ASV metadata and changes it when
 * it gets the chance.
 */
@Test public void testOldPermissionMetadata() throws Exception {
  Path selfishFile=new Path("/noOneElse");
  HashMap metadata=new HashMap();
  metadata.put("asv_permission",getExpectedPermissionString("rw-------"));
  backingStore.setContent(AzureBlobStorageTestAccount.toMockUri(selfishFile),new byte[]{},metadata);
  FsPermission justMe=new FsPermission(FsAction.READ_WRITE,FsAction.NONE,FsAction.NONE);
  FileStatus retrievedStatus=fs.getFileStatus(selfishFile);
  assertNotNull(retrievedStatus);
  assertEquals(justMe,retrievedStatus.getPermission());
  assertEquals(getExpectedOwner(),retrievedStatus.getOwner());
  assertEquals(NativeAzureFileSystem.AZURE_DEFAULT_GROUP_DEFAULT,retrievedStatus.getGroup());
  FsPermission meAndYou=new FsPermission(FsAction.READ_WRITE,FsAction.READ_WRITE,FsAction.NONE);
  fs.setPermission(selfishFile,meAndYou);
  metadata=backingStore.getMetadata(AzureBlobStorageTestAccount.toMockUri(selfishFile));
  assertNotNull(metadata);
  String storedPermission=metadata.get("hdi_permission");
  assertEquals(getExpectedPermissionString("rw-rw----"),storedPermission);
  assertNull(metadata.get("asv_permission"));
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testFolderMetadata() throws Exception {
  Path folder=new Path("/folder");
  FsPermission justRead=new FsPermission(FsAction.READ,FsAction.READ,FsAction.READ);
  fs.mkdirs(folder,justRead);
  HashMap metadata=backingStore.getMetadata(AzureBlobStorageTestAccount.toMockUri(folder));
  assertNotNull(metadata);
  assertEquals("true",metadata.get("hdi_isfolder"));
  assertEquals(getExpectedPermissionString("r--r--r--"),metadata.get("hdi_permission"));
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@SuppressWarnings("deprecation") @Test public void testPermissionMetadata() throws Exception {
  FsPermission justMe=new FsPermission(FsAction.READ_WRITE,FsAction.NONE,FsAction.NONE);
  Path selfishFile=new Path("/noOneElse");
  fs.create(selfishFile,justMe,true,4096,fs.getDefaultReplication(),fs.getDefaultBlockSize(),null).close();
  HashMap metadata=backingStore.getMetadata(AzureBlobStorageTestAccount.toMockUri(selfishFile));
  assertNotNull(metadata);
  String storedPermission=metadata.get("hdi_permission");
  assertEquals(getExpectedPermissionString("rw-------"),storedPermission);
  FileStatus retrievedStatus=fs.getFileStatus(selfishFile);
  assertNotNull(retrievedStatus);
  assertEquals(justMe,retrievedStatus.getPermission());
  assertEquals(getExpectedOwner(),retrievedStatus.getOwner());
  assertEquals(NativeAzureFileSystem.AZURE_DEFAULT_GROUP_DEFAULT,retrievedStatus.getGroup());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests that WASB works well with an older version container with ASV-era
 * version and metadata.
 */
@Test public void testFirstContainerVersionMetadata() throws Exception {
  HashMap containerMetadata=new HashMap();
  containerMetadata.put(AzureNativeFileSystemStore.OLD_VERSION_METADATA_KEY,AzureNativeFileSystemStore.FIRST_WASB_VERSION);
  FsWithPreExistingContainer fsWithContainer=FsWithPreExistingContainer.create(containerMetadata);
  assertFalse(fsWithContainer.getFs().exists(new Path("/IDontExist")));
  assertEquals(0,fsWithContainer.getFs().listStatus(new Path("/")).length);
  assertEquals(AzureNativeFileSystemStore.FIRST_WASB_VERSION,fsWithContainer.getContainerMetadata().get(AzureNativeFileSystemStore.OLD_VERSION_METADATA_KEY));
  assertNull(fsWithContainer.getContainerMetadata().get(AzureNativeFileSystemStore.VERSION_METADATA_KEY));
  fsWithContainer.getFs().mkdirs(new Path("/dir"));
  assertEquals(AzureNativeFileSystemStore.CURRENT_WASB_VERSION,fsWithContainer.getContainerMetadata().get(AzureNativeFileSystemStore.VERSION_METADATA_KEY));
  assertNull(fsWithContainer.getContainerMetadata().get(AzureNativeFileSystemStore.OLD_VERSION_METADATA_KEY));
  fsWithContainer.close();
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests that WASB stamped the version in the container metadata if it does a
 * write operation to a pre-existing container.
 */
@Test public void testPreExistingContainerVersionMetadata() throws Exception {
  FsWithPreExistingContainer fsWithContainer=FsWithPreExistingContainer.create();
  assertFalse(fsWithContainer.getFs().exists(new Path("/IDontExist")));
  assertEquals(0,fsWithContainer.getFs().listStatus(new Path("/")).length);
  assertNull(fsWithContainer.getContainerMetadata());
  fsWithContainer.getFs().mkdirs(new Path("/dir"));
  assertNotNull(fsWithContainer.getContainerMetadata());
  assertEquals(AzureNativeFileSystemStore.CURRENT_WASB_VERSION,fsWithContainer.getContainerMetadata().get(AzureNativeFileSystemStore.VERSION_METADATA_KEY));
  fsWithContainer.close();
}

Class: org.apache.hadoop.fs.azure.TestNativeAzureFileSystemBlockLocations

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testNumberOfBlocks() throws Exception {
  Configuration conf=new Configuration();
  conf.set(NativeAzureFileSystem.AZURE_BLOCK_SIZE_PROPERTY_NAME,"500");
  AzureBlobStorageTestAccount testAccount=AzureBlobStorageTestAccount.createMock(conf);
  FileSystem fs=testAccount.getFileSystem();
  Path testFile=createTestFile(fs,1200);
  FileStatus stat=fs.getFileStatus(testFile);
  assertEquals(500,stat.getBlockSize());
  testAccount.cleanup();
}

Class: org.apache.hadoop.fs.azure.TestNativeAzureFileSystemConcurrency

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier HybridVerifier 

@Test public void testLinkBlobs() throws Exception {
  Path filePath=new Path("/inProgress");
  FSDataOutputStream outputStream=fs.create(filePath);
  HashMap metadata=backingStore.getMetadata(AzureBlobStorageTestAccount.toMockUri(filePath));
  assertNotNull(metadata);
  String linkValue=metadata.get(AzureNativeFileSystemStore.LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
  assertNotNull(linkValue);
  assertTrue(backingStore.exists(AzureBlobStorageTestAccount.toMockUri(linkValue)));
  assertTrue(fs.exists(filePath));
  outputStream.close();
  metadata=backingStore.getMetadata(AzureBlobStorageTestAccount.toMockUri(filePath));
  assertNull(metadata.get(AzureNativeFileSystemStore.LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY));
}

Class: org.apache.hadoop.fs.azure.TestWasbUriAndConfiguration

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testDefaultKeyProvider() throws Exception {
  Configuration conf=new Configuration();
  String account="testacct";
  String key="testkey";
  conf.set(SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account,key);
  String result=AzureNativeFileSystemStore.getAccountKeyFromConfiguration(account,conf);
  assertEquals(key,result);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testValidKeyProvider() throws Exception {
  Configuration conf=new Configuration();
  String account="testacct";
  String key="testkey";
  conf.set(SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account,key);
  conf.setClass("fs.azure.account.keyprovider." + account,SimpleKeyProvider.class,KeyProvider.class);
  String result=AzureNativeFileSystemStore.getAccountKeyFromConfiguration(account,conf);
  assertEquals(key,result);
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests the cases when the URI is specified with no authority, i.e.
 * wasb:///path/to/file.
 */
@Test public void testNoUriAuthority() throws Exception {
  String[] wasbAliases=new String[]{"wasb","wasbs"};
  for (  String defaultScheme : wasbAliases) {
    for (    String wantedScheme : wasbAliases) {
      testAccount=AzureBlobStorageTestAccount.createMock();
      Configuration conf=testAccount.getFileSystem().getConf();
      String authority=testAccount.getFileSystem().getUri().getAuthority();
      URI defaultUri=new URI(defaultScheme,authority,null,null,null);
      conf.set("fs.default.name",defaultUri.toString());
      URI wantedUri=new URI(wantedScheme + ":///random/path");
      NativeAzureFileSystem obtained=(NativeAzureFileSystem)FileSystem.get(wantedUri,conf);
      assertNotNull(obtained);
      assertEquals(new URI(wantedScheme,authority,null,null,null),obtained.getUri());
      Path qualified=obtained.makeQualified(new Path(wantedUri));
      assertEquals(new URI(wantedScheme,authority,wantedUri.getPath(),null,null),qualified.toUri());
      testAccount.cleanup();
      FileSystem.closeAll();
    }
  }
  testAccount=AzureBlobStorageTestAccount.createMock();
  Configuration conf=testAccount.getFileSystem().getConf();
  conf.set("fs.default.name","file:///");
  try {
    FileSystem.get(new URI("wasb:///random/path"),conf);
    fail("Should've thrown.");
  }
 catch (  IllegalArgumentException e) {
  }
}

Class: org.apache.hadoop.fs.azure.metrics.TestAzureFileSystemInstrumentation

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testMetricsOnFileCreateRead() throws Exception {
  long base=getBaseWebResponses();
  assertEquals(0,AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation()));
  Path filePath=new Path("/metricsTest_webResponses");
  final int FILE_SIZE=1000;
  getBandwidthGaugeUpdater().suppressAutoUpdate();
  Date start=new Date();
  OutputStream outputStream=fs.create(filePath);
  outputStream.write(nonZeroByteArray(FILE_SIZE));
  outputStream.close();
  long uploadDurationMs=new Date().getTime() - start.getTime();
  logOpResponseCount("Creating a 1K file",base);
  base=assertWebResponsesInRange(base,2,15);
  getBandwidthGaugeUpdater().triggerUpdate(true);
  long bytesWritten=AzureMetricsTestUtil.getCurrentBytesWritten(getInstrumentation());
  assertTrue("The bytes written in the last second " + bytesWritten + " is pretty far from the expected range of around "+ FILE_SIZE+ " bytes plus a little overhead.",bytesWritten > (FILE_SIZE / 2) && bytesWritten < (FILE_SIZE * 2));
  long totalBytesWritten=AzureMetricsTestUtil.getCurrentTotalBytesWritten(getInstrumentation());
  assertTrue("The total bytes written  " + totalBytesWritten + " is pretty far from the expected range of around "+ FILE_SIZE+ " bytes plus a little overhead.",totalBytesWritten >= FILE_SIZE && totalBytesWritten < (FILE_SIZE * 2));
  long uploadRate=AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(),WASB_UPLOAD_RATE);
  System.out.println("Upload rate: " + uploadRate + " bytes/second.");
  long expectedRate=(FILE_SIZE * 1000L) / uploadDurationMs;
  assertTrue("The upload rate " + uploadRate + " is below the expected range of around "+ expectedRate+ " bytes/second that the unit test observed. This should never be"+ " the case since the test underestimates the rate by looking at "+ " end-to-end time instead of just block upload time.",uploadRate >= expectedRate);
  long uploadLatency=AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(),WASB_UPLOAD_LATENCY);
  System.out.println("Upload latency: " + uploadLatency);
  long expectedLatency=uploadDurationMs;
  assertTrue("The upload latency " + uploadLatency + " should be greater than zero now that I've just uploaded a file.",uploadLatency > 0);
  assertTrue("The upload latency " + uploadLatency + " is more than the expected range of around "+ expectedLatency+ " milliseconds that the unit test observed. This should never be"+ " the case since the test overestimates the latency by looking at "+ " end-to-end time instead of just block upload time.",uploadLatency <= expectedLatency);
  start=new Date();
  InputStream inputStream=fs.open(filePath);
  int count=0;
  while (inputStream.read() >= 0) {
    count++;
  }
  inputStream.close();
  long downloadDurationMs=new Date().getTime() - start.getTime();
  assertEquals(FILE_SIZE,count);
  logOpResponseCount("Reading a 1K file",base);
  base=assertWebResponsesInRange(base,1,10);
  getBandwidthGaugeUpdater().triggerUpdate(false);
  long totalBytesRead=AzureMetricsTestUtil.getCurrentTotalBytesRead(getInstrumentation());
  assertEquals(FILE_SIZE,totalBytesRead);
  long bytesRead=AzureMetricsTestUtil.getCurrentBytesRead(getInstrumentation());
  assertTrue("The bytes read in the last second " + bytesRead + " is pretty far from the expected range of around "+ FILE_SIZE+ " bytes plus a little overhead.",bytesRead > (FILE_SIZE / 2) && bytesRead < (FILE_SIZE * 2));
  long downloadRate=AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(),WASB_DOWNLOAD_RATE);
  System.out.println("Download rate: " + downloadRate + " bytes/second.");
  expectedRate=(FILE_SIZE * 1000L) / downloadDurationMs;
  assertTrue("The download rate " + downloadRate + " is below the expected range of around "+ expectedRate+ " bytes/second that the unit test observed. This should never be"+ " the case since the test underestimates the rate by looking at "+ " end-to-end time instead of just block download time.",downloadRate >= expectedRate);
  long downloadLatency=AzureMetricsTestUtil.getLongGaugeValue(getInstrumentation(),WASB_DOWNLOAD_LATENCY);
  System.out.println("Download latency: " + downloadLatency);
  expectedLatency=downloadDurationMs;
  assertTrue("The download latency " + downloadLatency + " should be greater than zero now that I've just downloaded a file.",downloadLatency > 0);
  assertTrue("The download latency " + downloadLatency + " is more than the expected range of around "+ expectedLatency+ " milliseconds that the unit test observed. This should never be"+ " the case since the test overestimates the latency by looking at "+ " end-to-end time instead of just block download time.",downloadLatency <= expectedLatency);
  assertNoErrors();
}

Class: org.apache.hadoop.fs.azure.metrics.TestBandwidthGaugeUpdater

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testSingleThreaded() throws Exception {
  AzureFileSystemInstrumentation instrumentation=new AzureFileSystemInstrumentation(new Configuration());
  BandwidthGaugeUpdater updater=new BandwidthGaugeUpdater(instrumentation,1000,true);
  updater.triggerUpdate(true);
  assertEquals(0,AzureMetricsTestUtil.getCurrentBytesWritten(instrumentation));
  updater.blockUploaded(new Date(),new Date(),150);
  updater.triggerUpdate(true);
  assertEquals(150,AzureMetricsTestUtil.getCurrentBytesWritten(instrumentation));
  updater.blockUploaded(new Date(new Date().getTime() - 10000),new Date(),200);
  updater.triggerUpdate(true);
  long currentBytes=AzureMetricsTestUtil.getCurrentBytesWritten(instrumentation);
  assertTrue("We expect around (200/10 = 20) bytes written as the gauge value." + "Got " + currentBytes,currentBytes > 18 && currentBytes < 22);
  updater.close();
}

Class: org.apache.hadoop.fs.contract.AbstractContractDeleteTest

APIUtilityVerifier UtilityVerifier 

@Test public void testDeleteNonEmptyDirNonRecursive() throws Throwable {
  Path path=path("testDeleteNonEmptyDirNonRecursive");
  mkdirs(path);
  Path file=new Path(path,"childfile");
  ContractTestUtils.writeTextFile(getFileSystem(),file,"goodbye, world",true);
  try {
    ContractTestUtils.rejectRootOperation(path);
    boolean deleted=getFileSystem().delete(path,false);
    fail("non recursive delete should have raised an exception," + " but completed with exit code " + deleted);
  }
 catch (  IOException expected) {
    handleExpectedException(expected);
  }
  ContractTestUtils.assertIsDirectory(getFileSystem(),path);
}

Class: org.apache.hadoop.fs.contract.AbstractContractOpenTest

APIUtilityVerifier EqualityVerifier 

@Test public void testOpenReadZeroByteFile() throws Throwable {
  describe("create & read a 0 byte file");
  Path path=path("zero.txt");
  touch(getFileSystem(),path);
  instream=getFileSystem().open(path);
  assertEquals(0,instream.getPos());
  int result=instream.read();
  assertMinusOne("initial byte read",result);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testOpenFileTwice() throws Throwable {
  describe("verify that two opened file streams are independent");
  Path path=path("testopenfiletwice.txt");
  byte[] block=dataset(TEST_FILE_LEN,0,255);
  createFile(getFileSystem(),path,false,block);
  FSDataInputStream instream1=getFileSystem().open(path);
  int c=instream1.read();
  assertEquals(0,c);
  FSDataInputStream instream2=null;
  try {
    instream2=getFileSystem().open(path);
    assertEquals("first read of instream 2",0,instream2.read());
    assertEquals("second read of instream 1",1,instream1.read());
    instream1.close();
    assertEquals("second read of instream 2",1,instream2.read());
    instream1.close();
  }
  finally {
    IOUtils.closeStream(instream1);
    IOUtils.closeStream(instream2);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testSequentialRead() throws Throwable {
  describe("verify that sequential read() operations return values");
  Path path=path("testsequentialread.txt");
  int len=4;
  int base=0x40;
  byte[] block=dataset(len,base,base + len);
  createFile(getFileSystem(),path,false,block);
  instream=getFileSystem().open(path);
  assertEquals(base,instream.read());
  assertEquals(base + 1,instream.read());
  assertEquals(base + 2,instream.read());
  assertEquals(base + 3,instream.read());
  assertEquals(-1,instream.read());
  assertEquals(-1,instream.read());
  instream.close();
}

Class: org.apache.hadoop.fs.contract.AbstractContractRenameTest

APIUtilityVerifier BooleanVerifier 

@Test public void testRenameDirIntoExistingDir() throws Throwable {
  describe("Verify renaming a dir into an existing dir puts it underneath" + " and leaves existing files alone");
  FileSystem fs=getFileSystem();
  String sourceSubdir="source";
  Path srcDir=path(sourceSubdir);
  Path srcFilePath=new Path(srcDir,"source-256.txt");
  byte[] srcDataset=dataset(256,'a','z');
  writeDataset(fs,srcFilePath,srcDataset,srcDataset.length,1024,false);
  Path destDir=path("dest");
  Path destFilePath=new Path(destDir,"dest-512.txt");
  byte[] destDateset=dataset(512,'A','Z');
  writeDataset(fs,destFilePath,destDateset,destDateset.length,1024,false);
  assertIsFile(destFilePath);
  boolean rename=rename(srcDir,destDir);
  Path renamedSrc=new Path(destDir,sourceSubdir);
  assertIsFile(destFilePath);
  assertIsDirectory(renamedSrc);
  ContractTestUtils.verifyFileContents(fs,destFilePath,destDateset);
  assertTrue("rename returned false though the contents were copied",rename);
}

APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier HybridVerifier 

/** 
 * Rename test -handles filesystems that will overwrite the destination
 * as well as those that do not (i.e. HDFS).
 * @throws Throwable
 */
@Test public void testRenameFileOverExistingFile() throws Throwable {
  describe("Verify renaming a file onto an existing file matches expectations");
  Path srcFile=path("source-256.txt");
  byte[] srcData=dataset(256,'a','z');
  writeDataset(getFileSystem(),srcFile,srcData,srcData.length,1024,false);
  Path destFile=path("dest-512.txt");
  byte[] destData=dataset(512,'A','Z');
  writeDataset(getFileSystem(),destFile,destData,destData.length,1024,false);
  assertIsFile(destFile);
  boolean renameOverwritesDest=isSupported(RENAME_OVERWRITES_DEST);
  boolean renameReturnsFalseOnRenameDestExists=!isSupported(RENAME_RETURNS_FALSE_IF_DEST_EXISTS);
  boolean destUnchanged=true;
  try {
    boolean renamed=rename(srcFile,destFile);
    if (renameOverwritesDest) {
      assertTrue("Rename returned false",renamed);
      destUnchanged=false;
    }
 else {
      if (renamed && !renameReturnsFalseOnRenameDestExists) {
        String destDirLS=generateAndLogErrorListing(srcFile,destFile);
        getLog().error("dest dir {}",destDirLS);
        fail("expected rename(" + srcFile + ", "+ destFile+ " ) to fail,"+ " but got success and destination of "+ destDirLS);
      }
    }
  }
 catch (  FileAlreadyExistsException e) {
    handleExpectedException(e);
  }
  ContractTestUtils.verifyFileContents(getFileSystem(),destFile,destUnchanged ? destData : srcData);
}

APIUtilityVerifier BooleanVerifier 

@Test public void testRenameNewFileSameDir() throws Throwable {
  describe("rename a file into a new file in the same directory");
  Path renameSrc=path("rename_src");
  Path renameTarget=path("rename_dest");
  byte[] data=dataset(256,'a','z');
  writeDataset(getFileSystem(),renameSrc,data,data.length,1024 * 1024,false);
  boolean rename=rename(renameSrc,renameTarget);
  assertTrue("rename(" + renameSrc + ", "+ renameTarget+ ") returned false",rename);
  ContractTestUtils.assertListStatusFinds(getFileSystem(),renameTarget.getParent(),renameTarget);
  ContractTestUtils.verifyFileContents(getFileSystem(),renameTarget,data);
}

APIUtilityVerifier BranchVerifier BooleanVerifier 

@Test public void testRenameFileNonexistentDir() throws Throwable {
  describe("rename a file into a new file in the same directory");
  Path renameSrc=path("testRenameSrc");
  Path renameTarget=path("subdir/testRenameTarget");
  byte[] data=dataset(256,'a','z');
  writeDataset(getFileSystem(),renameSrc,data,data.length,1024 * 1024,false);
  boolean renameCreatesDestDirs=isSupported(RENAME_CREATES_DEST_DIRS);
  try {
    boolean rename=rename(renameSrc,renameTarget);
    if (renameCreatesDestDirs) {
      assertTrue(rename);
      ContractTestUtils.verifyFileContents(getFileSystem(),renameTarget,data);
    }
 else {
      assertFalse(rename);
      ContractTestUtils.verifyFileContents(getFileSystem(),renameSrc,data);
    }
  }
 catch (  FileNotFoundException e) {
    assertFalse(renameCreatesDestDirs);
  }
}

APIUtilityVerifier BranchVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

@Test public void testRenameNonexistentFile() throws Throwable {
  describe("rename a file into a new file in the same directory");
  Path missing=path("testRenameNonexistentFileSrc");
  Path target=path("testRenameNonexistentFileDest");
  boolean renameReturnsFalseOnFailure=isSupported(ContractOptions.RENAME_RETURNS_FALSE_IF_SOURCE_MISSING);
  mkdirs(missing.getParent());
  try {
    boolean renamed=rename(missing,target);
    if (!renameReturnsFalseOnFailure) {
      String destDirLS=generateAndLogErrorListing(missing,target);
      fail("expected rename(" + missing + ", "+ target+ " ) to fail,"+ " got a result of "+ renamed+ " and a destination directory of "+ destDirLS);
    }
 else {
      getLog().warn("Rename returned {} renaming a nonexistent file",renamed);
      assertFalse("Renaming a missing file returned true",renamed);
    }
  }
 catch (  FileNotFoundException e) {
    if (renameReturnsFalseOnFailure) {
      ContractTestUtils.fail("Renaming a missing file unexpectedly threw an exception",e);
    }
    handleExpectedException(e);
  }
catch (  IOException e) {
    handleRelaxedException("rename nonexistent file","FileNotFoundException",e);
  }
  assertPathDoesNotExist("rename nonexistent file created a destination file",target);
}

Class: org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest

APIUtilityVerifier UtilityVerifier 

@Test public void testRmNonEmptyRootDirNonRecursive() throws Throwable {
  skipIfUnsupported(TEST_ROOT_TESTS_ENABLED);
  Path root=new Path("/");
  String touchfile="/testRmNonEmptyRootDirNonRecursive";
  Path file=new Path(touchfile);
  ContractTestUtils.touch(getFileSystem(),file);
  ContractTestUtils.assertIsDirectory(getFileSystem(),root);
  try {
    boolean deleted=getFileSystem().delete(root,false);
    fail("non recursive delete should have raised an exception," + " but completed with exit code " + deleted);
  }
 catch (  IOException e) {
    handleExpectedException(e);
  }
 finally {
    getFileSystem().delete(file,false);
  }
  ContractTestUtils.assertIsDirectory(getFileSystem(),root);
}

Class: org.apache.hadoop.fs.contract.AbstractContractSeekTest

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Seek round a file bigger than IO buffers
 * @throws Throwable
 */
@Test public void testSeekBigFile() throws Throwable {
  describe("Seek round a large file and verify the bytes are what is expected");
  Path testSeekFile=path("bigseekfile.txt");
  byte[] block=dataset(65536,0,255);
  createFile(getFileSystem(),testSeekFile,false,block);
  instream=getFileSystem().open(testSeekFile);
  assertEquals(0,instream.getPos());
  instream.seek(0);
  int result=instream.read();
  assertEquals(0,result);
  assertEquals(1,instream.read());
  assertEquals(2,instream.read());
  instream.seek(32768);
  assertEquals("@32768",block[32768],(byte)instream.read());
  instream.seek(40000);
  assertEquals("@40000",block[40000],(byte)instream.read());
  instream.seek(8191);
  assertEquals("@8191",block[8191],(byte)instream.read());
  instream.seek(0);
  assertEquals("@0",0,(byte)instream.read());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testSeekAndReadPastEndOfFile() throws Throwable {
  describe("verify that reading past the last bytes in the file returns -1");
  instream=getFileSystem().open(smallSeekFile);
  assertEquals(0,instream.getPos());
  instream.seek(TEST_FILE_LEN - 2);
  assertTrue("Premature EOF",instream.read() != -1);
  assertTrue("Premature EOF",instream.read() != -1);
  assertMinusOne("read past end of file",instream.read());
}

APIUtilityVerifier BranchVerifier UtilityVerifier InternalCallVerifier 

/** 
 * Seek and read on a closed file.
 * Some filesystems let callers seek on a closed file -these must
 * still fail on the subsequent reads.
 * @throws Throwable
 */
@Test public void testSeekReadClosedFile() throws Throwable {
  boolean supportsSeekOnClosedFiles=isSupported(SUPPORTS_SEEK_ON_CLOSED_FILE);
  instream=getFileSystem().open(smallSeekFile);
  getLog().debug("Stream is of type " + instream.getClass().getCanonicalName());
  instream.close();
  try {
    instream.seek(0);
    if (!supportsSeekOnClosedFiles) {
      fail("seek succeeded on a closed stream");
    }
  }
 catch (  IOException e) {
  }
  try {
    int data=instream.available();
    fail("read() succeeded on a closed stream, got " + data);
  }
 catch (  IOException e) {
  }
  try {
    int data=instream.read();
    fail("read() succeeded on a closed stream, got " + data);
  }
 catch (  IOException e) {
  }
  try {
    byte[] buffer=new byte[1];
    int result=instream.read(buffer,0,1);
    fail("read(buffer, 0, 1) succeeded on a closed stream, got " + result);
  }
 catch (  IOException e) {
  }
  try {
    long offset=instream.getPos();
  }
 catch (  IOException e) {
  }
  instream.close();
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testPositionedBulkReadDoesntChangePosition() throws Throwable {
  describe("verify that a positioned read does not change the getPos() value");
  Path testSeekFile=path("bigseekfile.txt");
  byte[] block=dataset(65536,0,255);
  createFile(getFileSystem(),testSeekFile,false,block);
  instream=getFileSystem().open(testSeekFile);
  instream.seek(39999);
  assertTrue(-1 != instream.read());
  assertEquals(40000,instream.getPos());
  byte[] readBuffer=new byte[256];
  instream.read(128,readBuffer,0,readBuffer.length);
  assertEquals(40000,instream.getPos());
  assertEquals("@40000",block[40000],(byte)instream.read());
  for (int i=0; i < 256; i++) {
    assertEquals("@" + i,block[i + 128],readBuffer[i]);
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testSeekPastEndOfFileThenReseekAndRead() throws Throwable {
  describe("do a seek past the EOF, then verify the stream recovers");
  instream=getFileSystem().open(smallSeekFile);
  boolean canSeekPastEOF=!getContract().isSupported(ContractOptions.REJECTS_SEEK_PAST_EOF,true);
  try {
    instream.seek(TEST_FILE_LEN + 1);
    assertMinusOne("read after seeking past EOF",instream.read());
  }
 catch (  EOFException e) {
    if (canSeekPastEOF) {
      throw e;
    }
    handleExpectedException(e);
  }
catch (  IOException e) {
    if (canSeekPastEOF) {
      throw e;
    }
    handleRelaxedException("a seek past the end of the file","EOFException",e);
  }
  instream.seek(1);
  assertTrue("Premature EOF",instream.read() != -1);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testSeekFile() throws Throwable {
  describe("basic seek operations");
  instream=getFileSystem().open(smallSeekFile);
  assertEquals(0,instream.getPos());
  instream.seek(0);
  int result=instream.read();
  assertEquals(0,result);
  assertEquals(1,instream.read());
  assertEquals(2,instream.getPos());
  assertEquals(2,instream.read());
  assertEquals(3,instream.getPos());
  instream.seek(128);
  assertEquals(128,instream.getPos());
  assertEquals(128,instream.read());
  instream.seek(63);
  assertEquals(63,instream.read());
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

@Test public void testNegativeSeek() throws Throwable {
  instream=getFileSystem().open(smallSeekFile);
  assertEquals(0,instream.getPos());
  try {
    instream.seek(-1);
    long p=instream.getPos();
    LOG.warn("Seek to -1 returned a position of " + p);
    int result=instream.read();
    fail("expected an exception, got data " + result + " at a position of "+ p);
  }
 catch (  EOFException e) {
    handleExpectedException(e);
  }
catch (  IOException e) {
    handleRelaxedException("a negative seek","EOFException",e);
  }
  assertEquals(0,instream.getPos());
}

APIUtilityVerifier EqualityVerifier 

@Test public void testSeekZeroByteFile() throws Throwable {
  describe("seek and read a 0 byte file");
  instream=getFileSystem().open(zeroByteFile);
  assertEquals(0,instream.getPos());
  int result=instream.read();
  assertMinusOne("initial byte read",result);
  byte[] buffer=new byte[1];
  instream.seek(0);
  result=instream.read();
  assertMinusOne("post-seek byte read",result);
  result=instream.read(buffer,0,1);
  assertMinusOne("post-seek buffer read",result);
}

APIUtilityVerifier EqualityVerifier 

@Test public void testBlockReadZeroByteFile() throws Throwable {
  describe("do a block read on a 0 byte file");
  instream=getFileSystem().open(zeroByteFile);
  assertEquals(0,instream.getPos());
  byte[] buffer=new byte[1];
  int result=instream.read(buffer,0,1);
  assertMinusOne("block read zero byte file",result);
}

Class: org.apache.hadoop.fs.http.server.TestHttpFSServer

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test ACL operations on a directory, including default ACLs.
 * General strategy is to use GETFILESTATUS and GETACLSTATUS to verify:
 * 

 * 
Initial status with no ACLs
 * 
The addition of a default ACL
 * 
The removal of default ACLs
 * 
 * @throws Exception
 */
@Test @TestDir @TestJetty @TestHdfs public void testDirAcls() throws Exception {
  final String defUser1="default:user:glarch:r-x";
  final String defSpec1="aclspec=" + defUser1;
  final String dir="/aclDirTest";
  String statusJson;
  List aclEntries;
  createHttpFSServer(false);
  FileSystem fs=FileSystem.get(TestHdfsHelper.getHdfsConf());
  fs.mkdirs(new Path(dir));
  statusJson=getStatus(dir,"GETFILESTATUS");
  Assert.assertEquals(-1,statusJson.indexOf("aclBit"));
  statusJson=getStatus(dir,"GETACLSTATUS");
  aclEntries=getAclEntries(statusJson);
  Assert.assertTrue(aclEntries.size() == 0);
  putCmd(dir,"SETACL",defSpec1);
  statusJson=getStatus(dir,"GETFILESTATUS");
  Assert.assertNotEquals(-1,statusJson.indexOf("aclBit"));
  statusJson=getStatus(dir,"GETACLSTATUS");
  aclEntries=getAclEntries(statusJson);
  Assert.assertTrue(aclEntries.size() == 5);
  Assert.assertTrue(aclEntries.contains(defUser1));
  putCmd(dir,"REMOVEDEFAULTACL",null);
  statusJson=getStatus(dir,"GETFILESTATUS");
  Assert.assertEquals(-1,statusJson.indexOf("aclBit"));
  statusJson=getStatus(dir,"GETACLSTATUS");
  aclEntries=getAclEntries(statusJson);
  Assert.assertTrue(aclEntries.size() == 0);
}

APIUtilityVerifier EqualityVerifier 

@Test @TestDir @TestJetty @TestHdfs public void testGlobFilter() throws Exception {
  createHttpFSServer(false);
  FileSystem fs=FileSystem.get(TestHdfsHelper.getHdfsConf());
  fs.mkdirs(new Path("/tmp"));
  fs.create(new Path("/tmp/foo.txt")).close();
  String user=HadoopUsersConfTestHelper.getHadoopUsers()[0];
  URL url=new URL(TestJettyHelper.getJettyURL(),MessageFormat.format("/webhdfs/v1/tmp?user.name={0}&op=liststatus&filter=f*",user));
  HttpURLConnection conn=(HttpURLConnection)url.openConnection();
  Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_OK);
  BufferedReader reader=new BufferedReader(new InputStreamReader(conn.getInputStream()));
  reader.readLine();
  reader.close();
}

APIUtilityVerifier EqualityVerifier 

@Test @TestDir @TestJetty @TestHdfs public void testPutNoOperation() throws Exception {
  createHttpFSServer(false);
  String user=HadoopUsersConfTestHelper.getHadoopUsers()[0];
  URL url=new URL(TestJettyHelper.getJettyURL(),MessageFormat.format("/webhdfs/v1/foo?user.name={0}",user));
  HttpURLConnection conn=(HttpURLConnection)url.openConnection();
  conn.setDoInput(true);
  conn.setDoOutput(true);
  conn.setRequestMethod("PUT");
  Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_BAD_REQUEST);
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test @TestDir @TestJetty @TestHdfs public void instrumentation() throws Exception {
  createHttpFSServer(false);
  URL url=new URL(TestJettyHelper.getJettyURL(),MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation","nobody"));
  HttpURLConnection conn=(HttpURLConnection)url.openConnection();
  Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_UNAUTHORIZED);
  url=new URL(TestJettyHelper.getJettyURL(),MessageFormat.format("/webhdfs/v1?user.name={0}&op=instrumentation",HadoopUsersConfTestHelper.getHadoopUsers()[0]));
  conn=(HttpURLConnection)url.openConnection();
  Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_OK);
  BufferedReader reader=new BufferedReader(new InputStreamReader(conn.getInputStream()));
  String line=reader.readLine();
  reader.close();
  Assert.assertTrue(line.contains("\"counters\":{"));
  url=new URL(TestJettyHelper.getJettyURL(),MessageFormat.format("/webhdfs/v1/foo?user.name={0}&op=instrumentation",HadoopUsersConfTestHelper.getHadoopUsers()[0]));
  conn=(HttpURLConnection)url.openConnection();
  Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_BAD_REQUEST);
}

APIUtilityVerifier EqualityVerifier 

@Test @TestDir @TestJetty @TestHdfs public void testOpenOffsetLength() throws Exception {
  createHttpFSServer(false);
  byte[] array=new byte[]{0,1,2,3};
  FileSystem fs=FileSystem.get(TestHdfsHelper.getHdfsConf());
  fs.mkdirs(new Path("/tmp"));
  OutputStream os=fs.create(new Path("/tmp/foo"));
  os.write(array);
  os.close();
  String user=HadoopUsersConfTestHelper.getHadoopUsers()[0];
  URL url=new URL(TestJettyHelper.getJettyURL(),MessageFormat.format("/webhdfs/v1/tmp/foo?user.name={0}&op=open&offset=1&length=2",user));
  HttpURLConnection conn=(HttpURLConnection)url.openConnection();
  Assert.assertEquals(HttpURLConnection.HTTP_OK,conn.getResponseCode());
  InputStream is=conn.getInputStream();
  Assert.assertEquals(1,is.read());
  Assert.assertEquals(2,is.read());
  Assert.assertEquals(-1,is.read());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test @TestDir @TestJetty @TestHdfs public void testDelegationTokenOperations() throws Exception {
  createHttpFSServer(true);
  URL url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=GETHOMEDIRECTORY");
  HttpURLConnection conn=(HttpURLConnection)url.openConnection();
  Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,conn.getResponseCode());
  AuthenticationToken token=new AuthenticationToken("u","p",new KerberosDelegationTokenAuthenticationHandler().getType());
  token.setExpires(System.currentTimeMillis() + 100000000);
  Signer signer=new Signer(new StringSignerSecretProvider("secret"));
  String tokenSigned=signer.sign(token.toString());
  url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=GETHOMEDIRECTORY");
  conn=(HttpURLConnection)url.openConnection();
  conn.setRequestProperty("Cookie",AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
  Assert.assertEquals(HttpURLConnection.HTTP_OK,conn.getResponseCode());
  url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=GETDELEGATIONTOKEN");
  conn=(HttpURLConnection)url.openConnection();
  conn.setRequestProperty("Cookie",AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
  Assert.assertEquals(HttpURLConnection.HTTP_OK,conn.getResponseCode());
  JSONObject json=(JSONObject)new JSONParser().parse(new InputStreamReader(conn.getInputStream()));
  json=(JSONObject)json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_JSON);
  String tokenStr=(String)json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON);
  url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
  conn=(HttpURLConnection)url.openConnection();
  Assert.assertEquals(HttpURLConnection.HTTP_OK,conn.getResponseCode());
  url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
  conn=(HttpURLConnection)url.openConnection();
  conn.setRequestMethod("PUT");
  Assert.assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED,conn.getResponseCode());
  url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
  conn=(HttpURLConnection)url.openConnection();
  conn.setRequestMethod("PUT");
  conn.setRequestProperty("Cookie",AuthenticatedURL.AUTH_COOKIE + "=" + tokenSigned);
  Assert.assertEquals(HttpURLConnection.HTTP_OK,conn.getResponseCode());
  url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=CANCELDELEGATIONTOKEN&token=" + tokenStr);
  conn=(HttpURLConnection)url.openConnection();
  conn.setRequestMethod("PUT");
  Assert.assertEquals(HttpURLConnection.HTTP_OK,conn.getResponseCode());
  url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
  conn=(HttpURLConnection)url.openConnection();
  Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN,conn.getResponseCode());
}

APIUtilityVerifier EqualityVerifier 

/** 
 * Validate XAttr get/set/remove calls.
 */
@Test @TestDir @TestJetty @TestHdfs public void testXAttrs() throws Exception {
  final String name1="user.a1";
  final byte[] value1=new byte[]{0x31,0x32,0x33};
  final String name2="user.a2";
  final byte[] value2=new byte[]{0x41,0x42,0x43};
  final String dir="/xattrTest";
  final String path=dir + "/file";
  createHttpFSServer(false);
  FileSystem fs=FileSystem.get(TestHdfsHelper.getHdfsConf());
  fs.mkdirs(new Path(dir));
  createWithHttp(path,null);
  String statusJson=getStatus(path,"GETXATTRS");
  Map xAttrs=getXAttrs(statusJson);
  Assert.assertEquals(0,xAttrs.size());
  putCmd(path,"SETXATTR",setXAttrParam(name1,value1));
  putCmd(path,"SETXATTR",setXAttrParam(name2,value2));
  statusJson=getStatus(path,"GETXATTRS");
  xAttrs=getXAttrs(statusJson);
  Assert.assertEquals(2,xAttrs.size());
  Assert.assertArrayEquals(value1,xAttrs.get(name1));
  Assert.assertArrayEquals(value2,xAttrs.get(name2));
  putCmd(path,"REMOVEXATTR","xattr.name=" + name1);
  statusJson=getStatus(path,"GETXATTRS");
  xAttrs=getXAttrs(statusJson);
  Assert.assertEquals(1,xAttrs.size());
  Assert.assertArrayEquals(value2,xAttrs.get(name2));
  putCmd(path,"REMOVEXATTR","xattr.name=" + name2);
  statusJson=getStatus(path,"GETXATTRS");
  xAttrs=getXAttrs(statusJson);
  Assert.assertEquals(0,xAttrs.size());
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Validate the various ACL set/modify/remove calls.  General strategy is
 * to verify each of the following steps with GETFILESTATUS, LISTSTATUS,
 * and GETACLSTATUS:
 * 

 * 
Create a file with no ACLs
 * 
Add a user + group ACL
 * 
Add another user ACL
 * 
Remove the first user ACL
 * 
Remove all ACLs
 * 
 */
@Test @TestDir @TestJetty @TestHdfs public void testFileAcls() throws Exception {
  final String aclUser1="user:foo:rw-";
  final String aclUser2="user:bar:r--";
  final String aclGroup1="group::r--";
  final String aclSpec="aclspec=user::rwx," + aclUser1 + ","+ aclGroup1+ ",other::---";
  final String modAclSpec="aclspec=" + aclUser2;
  final String remAclSpec="aclspec=" + aclUser1;
  final String dir="/aclFileTest";
  final String path=dir + "/test";
  String statusJson;
  List aclEntries;
  createHttpFSServer(false);
  FileSystem fs=FileSystem.get(TestHdfsHelper.getHdfsConf());
  fs.mkdirs(new Path(dir));
  createWithHttp(path,null);
  statusJson=getStatus(path,"GETFILESTATUS");
  Assert.assertEquals(-1,statusJson.indexOf("aclBit"));
  statusJson=getStatus(dir,"LISTSTATUS");
  Assert.assertEquals(-1,statusJson.indexOf("aclBit"));
  statusJson=getStatus(path,"GETACLSTATUS");
  aclEntries=getAclEntries(statusJson);
  Assert.assertTrue(aclEntries.size() == 0);
  putCmd(path,"SETACL",aclSpec);
  statusJson=getStatus(path,"GETFILESTATUS");
  Assert.assertNotEquals(-1,statusJson.indexOf("aclBit"));
  statusJson=getStatus(dir,"LISTSTATUS");
  Assert.assertNotEquals(-1,statusJson.indexOf("aclBit"));
  statusJson=getStatus(path,"GETACLSTATUS");
  aclEntries=getAclEntries(statusJson);
  Assert.assertTrue(aclEntries.size() == 2);
  Assert.assertTrue(aclEntries.contains(aclUser1));
  Assert.assertTrue(aclEntries.contains(aclGroup1));
  putCmd(path,"MODIFYACLENTRIES",modAclSpec);
  statusJson=getStatus(path,"GETACLSTATUS");
  aclEntries=getAclEntries(statusJson);
  Assert.assertTrue(aclEntries.size() == 3);
  Assert.assertTrue(aclEntries.contains(aclUser1));
  Assert.assertTrue(aclEntries.contains(aclUser2));
  Assert.assertTrue(aclEntries.contains(aclGroup1));
  putCmd(path,"REMOVEACLENTRIES",remAclSpec);
  statusJson=getStatus(path,"GETACLSTATUS");
  aclEntries=getAclEntries(statusJson);
  Assert.assertTrue(aclEntries.size() == 2);
  Assert.assertTrue(aclEntries.contains(aclUser2));
  Assert.assertTrue(aclEntries.contains(aclGroup1));
  putCmd(path,"REMOVEACL",null);
  statusJson=getStatus(path,"GETACLSTATUS");
  aclEntries=getAclEntries(statusJson);
  Assert.assertTrue(aclEntries.size() == 0);
  statusJson=getStatus(path,"GETFILESTATUS");
  Assert.assertEquals(-1,statusJson.indexOf("aclBit"));
  statusJson=getStatus(dir,"LISTSTATUS");
  Assert.assertEquals(-1,statusJson.indexOf("aclBit"));
}

APIUtilityVerifier EqualityVerifier 

@Test @TestDir @TestJetty @TestHdfs public void testHdfsAccess() throws Exception {
  createHttpFSServer(false);
  String user=HadoopUsersConfTestHelper.getHadoopUsers()[0];
  URL url=new URL(TestJettyHelper.getJettyURL(),MessageFormat.format("/webhdfs/v1/?user.name={0}&op=liststatus",user));
  HttpURLConnection conn=(HttpURLConnection)url.openConnection();
  Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_OK);
  BufferedReader reader=new BufferedReader(new InputStreamReader(conn.getInputStream()));
  reader.readLine();
  reader.close();
}

Class: org.apache.hadoop.fs.http.server.TestHttpFSWithKerberos

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test @TestDir @TestJetty @TestHdfs public void testDelegationTokenHttpFSAccess() throws Exception {
  createHttpFSServer();
  KerberosTestUtils.doAsClient(new Callable(){
    @Override public Void call() throws Exception {
      URL url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=GETDELEGATIONTOKEN");
      AuthenticatedURL aUrl=new AuthenticatedURL();
      AuthenticatedURL.Token aToken=new AuthenticatedURL.Token();
      HttpURLConnection conn=aUrl.openConnection(url,aToken);
      Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_OK);
      JSONObject json=(JSONObject)new JSONParser().parse(new InputStreamReader(conn.getInputStream()));
      json=(JSONObject)json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_JSON);
      String tokenStr=(String)json.get(DelegationTokenAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON);
      url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
      conn=(HttpURLConnection)url.openConnection();
      Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_OK);
      url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
      conn=(HttpURLConnection)url.openConnection();
      conn.setRequestMethod("PUT");
      Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_UNAUTHORIZED);
      url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=RENEWDELEGATIONTOKEN&token=" + tokenStr);
      conn=aUrl.openConnection(url,aToken);
      conn.setRequestMethod("PUT");
      Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_OK);
      url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=CANCELDELEGATIONTOKEN&token=" + tokenStr);
      conn=(HttpURLConnection)url.openConnection();
      conn.setRequestMethod("PUT");
      Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_OK);
      url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=GETHOMEDIRECTORY&delegation=" + tokenStr);
      conn=(HttpURLConnection)url.openConnection();
      Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_UNAUTHORIZED);
      return null;
    }
  }
);
}

APIUtilityVerifier EqualityVerifier 

@Test @TestDir @TestJetty @TestHdfs public void testInvalidadHttpFSAccess() throws Exception {
  createHttpFSServer();
  URL url=new URL(TestJettyHelper.getJettyURL(),"/webhdfs/v1/?op=GETHOMEDIRECTORY");
  HttpURLConnection conn=(HttpURLConnection)url.openConnection();
  Assert.assertEquals(conn.getResponseCode(),HttpURLConnection.HTTP_UNAUTHORIZED);
}

Class: org.apache.hadoop.fs.loadGenerator.TestLoadGenerator

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test if the structure generator works fine 
 */
@Test public void testStructureGenerator() throws Exception {
  StructureGenerator sg=new StructureGenerator();
  String[] args=new String[]{"-maxDepth","2","-minWidth","1","-maxWidth","2","-numOfFiles","2","-avgFileSize","1","-outDir",OUT_DIR.getAbsolutePath(),"-seed","1"};
  final int MAX_DEPTH=1;
  final int MIN_WIDTH=3;
  final int MAX_WIDTH=5;
  final int NUM_OF_FILES=7;
  final int AVG_FILE_SIZE=9;
  final int SEED=13;
  try {
    assertEquals(0,sg.run(args));
    BufferedReader in=new BufferedReader(new FileReader(DIR_STRUCTURE_FILE));
    assertEquals(DIR_STRUCTURE_FIRST_LINE,in.readLine());
    assertEquals(DIR_STRUCTURE_SECOND_LINE,in.readLine());
    assertEquals(null,in.readLine());
    in.close();
    in=new BufferedReader(new FileReader(FILE_STRUCTURE_FILE));
    assertEquals(FILE_STRUCTURE_FIRST_LINE,in.readLine());
    assertEquals(FILE_STRUCTURE_SECOND_LINE,in.readLine());
    assertEquals(null,in.readLine());
    in.close();
    String oldArg=args[MAX_DEPTH];
    args[MAX_DEPTH]="0";
    assertEquals(-1,sg.run(args));
    args[MAX_DEPTH]=oldArg;
    oldArg=args[MIN_WIDTH];
    args[MIN_WIDTH]="-1";
    assertEquals(-1,sg.run(args));
    args[MIN_WIDTH]=oldArg;
    oldArg=args[MAX_WIDTH];
    args[MAX_WIDTH]="-1";
    assertEquals(-1,sg.run(args));
    args[MAX_WIDTH]=oldArg;
    oldArg=args[NUM_OF_FILES];
    args[NUM_OF_FILES]="-1";
    assertEquals(-1,sg.run(args));
    args[NUM_OF_FILES]=oldArg;
    oldArg=args[NUM_OF_FILES];
    args[NUM_OF_FILES]="-1";
    assertEquals(-1,sg.run(args));
    args[NUM_OF_FILES]=oldArg;
    oldArg=args[AVG_FILE_SIZE];
    args[AVG_FILE_SIZE]="-1";
    assertEquals(-1,sg.run(args));
    args[AVG_FILE_SIZE]=oldArg;
    oldArg=args[SEED];
    args[SEED]="34.d4";
    assertEquals(-1,sg.run(args));
    args[SEED]=oldArg;
  }
  finally {
    DIR_STRUCTURE_FILE.delete();
    FILE_STRUCTURE_FILE.delete();
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test if the load generator works fine 
 */
@Test public void testLoadGenerator() throws Exception {
  final String TEST_SPACE_ROOT="/test";
  final String SCRIPT_TEST_DIR=OUT_DIR.getAbsolutePath();
  String script=SCRIPT_TEST_DIR + "/" + "loadgenscript";
  String script2=SCRIPT_TEST_DIR + "/" + "loadgenscript2";
  File scriptFile1=new File(script);
  File scriptFile2=new File(script2);
  FileWriter writer=new FileWriter(DIR_STRUCTURE_FILE);
  writer.write(DIR_STRUCTURE_FIRST_LINE + "\n");
  writer.write(DIR_STRUCTURE_SECOND_LINE + "\n");
  writer.close();
  writer=new FileWriter(FILE_STRUCTURE_FILE);
  writer.write(FILE_STRUCTURE_FIRST_LINE + "\n");
  writer.write(FILE_STRUCTURE_SECOND_LINE + "\n");
  writer.close();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(CONF).numDataNodes(3).build();
  cluster.waitActive();
  try {
    DataGenerator dg=new DataGenerator();
    dg.setConf(CONF);
    String[] args=new String[]{"-inDir",OUT_DIR.getAbsolutePath(),"-root",TEST_SPACE_ROOT};
    assertEquals(0,dg.run(args));
    final int READ_PROBABILITY=1;
    final int WRITE_PROBABILITY=3;
    final int MAX_DELAY_BETWEEN_OPS=7;
    final int NUM_OF_THREADS=9;
    final int START_TIME=11;
    final int ELAPSED_TIME=13;
    LoadGenerator lg=new LoadGenerator();
    lg.setConf(CONF);
    args=new String[]{"-readProbability","0.3","-writeProbability","0.3","-root",TEST_SPACE_ROOT,"-maxDelayBetweenOps","0","-numOfThreads","1","-startTime",Long.toString(Time.now()),"-elapsedTime","10"};
    assertEquals(0,lg.run(args));
    String oldArg=args[READ_PROBABILITY];
    args[READ_PROBABILITY]="1.1";
    assertEquals(-1,lg.run(args));
    args[READ_PROBABILITY]="-1.1";
    assertEquals(-1,lg.run(args));
    args[READ_PROBABILITY]=oldArg;
    oldArg=args[WRITE_PROBABILITY];
    args[WRITE_PROBABILITY]="1.1";
    assertEquals(-1,lg.run(args));
    args[WRITE_PROBABILITY]="-1.1";
    assertEquals(-1,lg.run(args));
    args[WRITE_PROBABILITY]="0.9";
    assertEquals(-1,lg.run(args));
    args[READ_PROBABILITY]=oldArg;
    oldArg=args[MAX_DELAY_BETWEEN_OPS];
    args[MAX_DELAY_BETWEEN_OPS]="1.x1";
    assertEquals(-1,lg.run(args));
    args[MAX_DELAY_BETWEEN_OPS]=oldArg;
    oldArg=args[MAX_DELAY_BETWEEN_OPS];
    args[MAX_DELAY_BETWEEN_OPS]="1.x1";
    assertEquals(-1,lg.run(args));
    args[MAX_DELAY_BETWEEN_OPS]=oldArg;
    oldArg=args[NUM_OF_THREADS];
    args[NUM_OF_THREADS]="-1";
    assertEquals(-1,lg.run(args));
    args[NUM_OF_THREADS]=oldArg;
    oldArg=args[START_TIME];
    args[START_TIME]="-1";
    assertEquals(-1,lg.run(args));
    args[START_TIME]=oldArg;
    oldArg=args[ELAPSED_TIME];
    args[ELAPSED_TIME]="-1";
    assertEquals(-1,lg.run(args));
    args[ELAPSED_TIME]=oldArg;
    FileWriter fw=new FileWriter(scriptFile1);
    fw.write("2 .22 .33\n");
    fw.write("3 .10 .6\n");
    fw.write("6 0 .7\n");
    fw.close();
    String[] scriptArgs=new String[]{"-root",TEST_SPACE_ROOT,"-maxDelayBetweenOps","0","-numOfThreads","10","-startTime",Long.toString(Time.now()),"-scriptFile",script};
    assertEquals(0,lg.run(scriptArgs));
    fw=new FileWriter(scriptFile2);
    fw.write("2 .22 .33\n");
    fw.write("3 blah blah blah .6\n");
    fw.write("6 0 .7\n");
    fw.close();
    scriptArgs[scriptArgs.length - 1]=script2;
    assertEquals(-1,lg.run(scriptArgs));
  }
  finally {
    cluster.shutdown();
    DIR_STRUCTURE_FILE.delete();
    FILE_STRUCTURE_FILE.delete();
    scriptFile1.delete();
    scriptFile2.delete();
  }
}

Class: org.apache.hadoop.fs.shell.TestHdfsTextCommand

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Tests whether binary Avro data files are displayed correctly.
 */
@Test public void testDisplayForAvroFiles() throws Exception {
  createAvroFile(generateWeatherAvroBinaryData());
  Configuration conf=fs.getConf();
  PathData pathData=new PathData(AVRO_FILENAME.toString(),conf);
  Display.Text text=new Display.Text();
  text.setConf(conf);
  Method method=text.getClass().getDeclaredMethod("getInputStream",PathData.class);
  method.setAccessible(true);
  InputStream stream=(InputStream)method.invoke(text,pathData);
  String output=inputStreamToString(stream);
  String expectedOutput="{\"station\":\"011990-99999\",\"time\":-619524000000,\"temp\":0}" + System.getProperty("line.separator") + "{\"station\":\"011990-99999\",\"time\":-619506000000,\"temp\":22}"+ System.getProperty("line.separator")+ "{\"station\":\"011990-99999\",\"time\":-619484400000,\"temp\":-11}"+ System.getProperty("line.separator")+ "{\"station\":\"012650-99999\",\"time\":-655531200000,\"temp\":111}"+ System.getProperty("line.separator")+ "{\"station\":\"012650-99999\",\"time\":-655509600000,\"temp\":78}"+ System.getProperty("line.separator");
  assertEquals(expectedOutput,output);
}

Class: org.apache.hadoop.fs.shell.TestPathData

APIUtilityVerifier EqualityVerifier 

@Test(timeout=30000) public void testRelativeGlob() throws Exception {
  PathData[] items=PathData.expandAsGlob("d1/f1*",conf);
  assertEquals(sortedString("d1/f1","d1/f1.1"),sortedString(items));
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=30000) public void testRelativeGlobBack() throws Exception {
  fs.setWorkingDirectory(new Path("d1"));
  PathData[] items=PathData.expandAsGlob("../d2/*",conf);
  assertEquals(sortedString("../d2/f3"),sortedString(items));
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=30000) public void testAbsoluteGlob() throws Exception {
  PathData[] items=PathData.expandAsGlob(testDir + "/d1/f1*",conf);
  assertEquals(sortedString(testDir + "/d1/f1",testDir + "/d1/f1.1"),sortedString(items));
  String absolutePathNoDriveLetter=testDir + "/d1/f1";
  if (Shell.WINDOWS) {
    absolutePathNoDriveLetter=absolutePathNoDriveLetter.substring(2);
  }
  items=PathData.expandAsGlob(absolutePathNoDriveLetter,conf);
  assertEquals(sortedString(absolutePathNoDriveLetter),sortedString(items));
  items=PathData.expandAsGlob(".",conf);
  assertEquals(sortedString("."),sortedString(items));
}

Class: org.apache.hadoop.fs.shell.TestTextCommand

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Tests whether binary Avro data files are displayed correctly.
 */
@Test(timeout=30000) public void testDisplayForAvroFiles() throws Exception {
  createAvroFile(generateWeatherAvroBinaryData());
  Configuration conf=new Configuration();
  URI localPath=new URI(AVRO_FILENAME);
  PathData pathData=new PathData(localPath,conf);
  Display.Text text=new Display.Text();
  text.setConf(conf);
  Method method=text.getClass().getDeclaredMethod("getInputStream",PathData.class);
  method.setAccessible(true);
  InputStream stream=(InputStream)method.invoke(text,pathData);
  String output=inputStreamToString(stream);
  String expectedOutput="{\"station\":\"011990-99999\",\"time\":-619524000000,\"temp\":0}" + System.getProperty("line.separator") + "{\"station\":\"011990-99999\",\"time\":-619506000000,\"temp\":22}"+ System.getProperty("line.separator")+ "{\"station\":\"011990-99999\",\"time\":-619484400000,\"temp\":-11}"+ System.getProperty("line.separator")+ "{\"station\":\"012650-99999\",\"time\":-655531200000,\"temp\":111}"+ System.getProperty("line.separator")+ "{\"station\":\"012650-99999\",\"time\":-655509600000,\"temp\":78}"+ System.getProperty("line.separator");
  assertEquals(expectedOutput,output);
}

Class: org.apache.hadoop.fs.slive.TestSlive

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testMRFlow() throws Exception {
  ConfigExtractor extractor=getTestConfig(false);
  SliveTest s=new SliveTest(getBaseConfig());
  int ec=ToolRunner.run(s,getTestArgs(false));
  assertTrue(ec == 0);
  String resFile=extractor.getResultFile();
  File fn=new File(resFile);
  assertTrue(fn.exists());
}

Class: org.apache.hadoop.fs.swift.TestReadPastBuffer

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Seek past the buffer then read
 * @throws Throwable problems
 */
@Test(timeout=SWIFT_TEST_TIMEOUT) public void testSeekAndReadPastEndOfFile() throws Throwable {
  instream=fs.open(readFile);
  assertEquals(0,instream.getPos());
  instream.seek(SEEK_FILE_LEN - 2);
  assertTrue("Premature EOF",instream.read() != -1);
  assertTrue("Premature EOF",instream.read() != -1);
  assertMinusOne("read past end of file",instream.read());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Seek past the buffer and attempt a read(buffer)
 * @throws Throwable failures
 */
@Test(timeout=SWIFT_TEST_TIMEOUT) public void testSeekBulkReadPastEndOfFile() throws Throwable {
  instream=fs.open(readFile);
  assertEquals(0,instream.getPos());
  instream.seek(SEEK_FILE_LEN - 1);
  byte[] buffer=new byte[1];
  int result=instream.read(buffer,0,1);
  result=instream.read(buffer,0,1);
  assertMinusOne("read past end of file",result);
  result=instream.read(buffer,0,1);
  assertMinusOne("read past end of file",result);
  result=instream.read(buffer,0,0);
  assertEquals("EOF checks coming before read range check",0,result);
}

Class: org.apache.hadoop.fs.swift.TestSeek

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testSeekAndPastEndOfFileThenReseekAndRead() throws Throwable {
  instream=fs.open(smallSeekFile);
  try {
    instream.seek(SMALL_SEEK_FILE_LEN);
    assertMinusOne("read after seeking past EOF",instream.read());
  }
 catch (  EOFException expected) {
  }
  instream.seek(1);
  assertTrue("Premature EOF",instream.read() != -1);
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testPositionedBulkReadDoesntChangePosition() throws Throwable {
  Path testSeekFile=new Path(testPath,"bigseekfile.txt");
  byte[] block=SwiftTestUtils.dataset(65536,0,255);
  createFile(testSeekFile,block);
  instream=fs.open(testSeekFile);
  instream.seek(39999);
  assertTrue(-1 != instream.read());
  assertEquals(40000,instream.getPos());
  byte[] readBuffer=new byte[256];
  instream.read(128,readBuffer,0,readBuffer.length);
  assertEquals(40000,instream.getPos());
  assertEquals("@40000",block[40000],(byte)instream.read());
  for (int i=0; i < 256; i++) {
    assertEquals("@" + i,block[i + 128],readBuffer[i]);
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testSeekAndReadPastEndOfFile() throws Throwable {
  instream=fs.open(smallSeekFile);
  assertEquals(0,instream.getPos());
  instream.seek(SMALL_SEEK_FILE_LEN - 2);
  assertTrue("Premature EOF",instream.read() != -1);
  assertTrue("Premature EOF",instream.read() != -1);
  assertMinusOne("read past end of file",instream.read());
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testNegativeSeek() throws Throwable {
  instream=fs.open(smallSeekFile);
  assertEquals(0,instream.getPos());
  try {
    instream.seek(-1);
    long p=instream.getPos();
    LOG.warn("Seek to -1 returned a position of " + p);
    int result=instream.read();
    fail("expected an exception, got data " + result + " at a position of "+ p);
  }
 catch (  IOException e) {
  }
  assertEquals(0,instream.getPos());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testSeekBigFile() throws Throwable {
  Path testSeekFile=new Path(testPath,"bigseekfile.txt");
  byte[] block=SwiftTestUtils.dataset(65536,0,255);
  createFile(testSeekFile,block);
  instream=fs.open(testSeekFile);
  assertEquals(0,instream.getPos());
  instream.seek(0);
  int result=instream.read();
  assertEquals(0,result);
  assertEquals(1,instream.read());
  assertEquals(2,instream.read());
  instream.seek(32768);
  assertEquals("@32768",block[32768],(byte)instream.read());
  instream.seek(40000);
  assertEquals("@40000",block[40000],(byte)instream.read());
  instream.seek(8191);
  assertEquals("@8191",block[8191],(byte)instream.read());
  instream.seek(0);
  assertEquals("@0",0,(byte)instream.read());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testSeekFile() throws Throwable {
  instream=fs.open(smallSeekFile);
  assertEquals(0,instream.getPos());
  instream.seek(0);
  int result=instream.read();
  assertEquals(0,result);
  assertEquals(1,instream.read());
  assertEquals(2,instream.getPos());
  assertEquals(2,instream.read());
  assertEquals(3,instream.getPos());
  instream.seek(128);
  assertEquals(128,instream.getPos());
  assertEquals(128,instream.read());
  instream.seek(63);
  assertEquals(63,instream.read());
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testSeekZeroByteFile() throws Throwable {
  instream=fs.open(zeroByteFile);
  assertEquals(0,instream.getPos());
  int result=instream.read();
  assertMinusOne("initial byte read",result);
  byte[] buffer=new byte[1];
  instream.seek(0);
  result=instream.read();
  assertMinusOne("post-seek byte read",result);
  result=instream.read(buffer,0,1);
  assertMinusOne("post-seek buffer read",result);
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testBlockReadZeroByteFile() throws Throwable {
  instream=fs.open(zeroByteFile);
  assertEquals(0,instream.getPos());
  byte[] buffer=new byte[1];
  int result=instream.read(buffer,0,1);
  assertMinusOne("block read zero byte file",result);
}

Class: org.apache.hadoop.fs.swift.TestSwiftConfig

APIUtilityVerifier BooleanVerifier 

@Test public void testLocationAwareFalsePropagates() throws Exception {
  final Configuration configuration=createCoreConfig();
  set(configuration,DOT_LOCATION_AWARE,"false");
  SwiftRestClient restClient=mkInstance(configuration);
  assertFalse(restClient.isLocationAware());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testProxyData() throws Exception {
  final Configuration configuration=createCoreConfig();
  String proxy="web-proxy";
  int port=8088;
  configuration.set(SWIFT_PROXY_HOST_PROPERTY,proxy);
  configuration.set(SWIFT_PROXY_PORT_PROPERTY,Integer.toString(port));
  SwiftRestClient restClient=mkInstance(configuration);
  assertEquals(proxy,restClient.getProxyHost());
  assertEquals(port,restClient.getProxyPort());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testPositivePartsize() throws Exception {
  final Configuration configuration=createCoreConfig();
  int size=127;
  configuration.set(SWIFT_PARTITION_SIZE,Integer.toString(size));
  SwiftRestClient restClient=mkInstance(configuration);
  assertEquals(size,restClient.getPartSizeKB());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testPositiveBlocksize() throws Exception {
  final Configuration configuration=createCoreConfig();
  int size=127;
  configuration.set(SWIFT_BLOCKSIZE,Integer.toString(size));
  SwiftRestClient restClient=mkInstance(configuration);
  assertEquals(size,restClient.getBlocksizeKB());
}

APIUtilityVerifier BooleanVerifier 

@Test public void testLocationAwareTruePropagates() throws Exception {
  final Configuration configuration=createCoreConfig();
  set(configuration,DOT_LOCATION_AWARE,"true");
  SwiftRestClient restClient=mkInstance(configuration);
  assertTrue(restClient.isLocationAware());
}

Class: org.apache.hadoop.fs.swift.TestSwiftFileSystemBasicOps

APIUtilityVerifier UtilityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testLongObjectNamesForbidden() throws Throwable {
  StringBuilder buffer=new StringBuilder(1200);
  buffer.append("/");
  for (int i=0; i < (1200 / 4); i++) {
    buffer.append(String.format("%04x",i));
  }
  String pathString=buffer.toString();
  Path path=new Path(pathString);
  try {
    writeTextFile(fs,path,pathString,true);
    fs.delete(path,false);
    fail("Managed to create an object with a name of length " + pathString.length());
  }
 catch (  SwiftBadRequestException e) {
  }
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testOverwrite() throws Throwable {
  Path path=new Path("/test/Overwrite");
  try {
    String text="Testing a put to a file " + System.currentTimeMillis();
    writeTextFile(fs,path,text,false);
    assertFileHasLength(fs,path,text.length());
    String text2="Overwriting a file " + System.currentTimeMillis();
    writeTextFile(fs,path,text2,true);
    assertFileHasLength(fs,path,text2.length());
    String result=readBytesToString(fs,path,text2.length());
    assertEquals(text2,result);
  }
  finally {
    delete(fs,path);
  }
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testPutGetFile() throws Throwable {
  Path path=new Path("/test/PutGetFile");
  try {
    String text="Testing a put and get to a file " + System.currentTimeMillis();
    writeTextFile(fs,path,text,false);
    String result=readBytesToString(fs,path,text.length());
    assertEquals(text,result);
  }
  finally {
    delete(fs,path);
  }
}

APIUtilityVerifier UtilityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testLsNonExistentFile() throws Exception {
  try {
    Path path=new Path("/test/hadoop/file");
    FileStatus[] statuses=fs.listStatus(path);
    fail("Should throw FileNotFoundException on " + path + " but got list of length "+ statuses.length);
  }
 catch (  FileNotFoundException fnfe) {
  }
}

Class: org.apache.hadoop.fs.swift.TestSwiftFileSystemBlocksize

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testBlocksizeNonZeroForFile() throws Throwable {
  Path smallfile=new Path("/test/smallfile");
  SwiftTestUtils.writeTextFile(fs,smallfile,"blocksize",true);
  createFile(smallfile);
  FileStatus status=getFs().getFileStatus(smallfile);
  assertTrue("Zero blocksize in " + status,status.getBlockSize() != 0L);
  assertTrue("Zero replication in " + status,status.getReplication() != 0L);
}

Class: org.apache.hadoop.fs.swift.TestSwiftFileSystemDirectories

APIUtilityVerifier BooleanVerifier 

/** 
 * Asserts that a zero byte file has a status of file and not
 * directory or symlink
 * @throws Exception on failures
 */
@Test(timeout=SWIFT_TEST_TIMEOUT) public void testMultiByteFilesAreFiles() throws Exception {
  Path src=path("/test/testMultiByteFilesAreFiles");
  SwiftTestUtils.writeTextFile(fs,src,"testMultiByteFilesAreFiles",false);
  assertIsFile(src);
  FileStatus status=fs.getFileStatus(src);
  assertFalse(status.isDir());
}

APIUtilityVerifier UtilityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testNoStatusForMissingDirectories() throws Throwable {
  Path missing=path("/test/testNoStatusForMissingDirectories");
  assertPathDoesNotExist("leftover?",missing);
  try {
    FileStatus[] statuses=fs.listStatus(missing);
    fail("Expected a FileNotFoundException, got the status " + statuses);
  }
 catch (  FileNotFoundException expected) {
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * test that a dir off root has a listStatus() call that
 * works as expected. and that when a child is added. it changes
 * @throws Exception on failures
 */
@Test(timeout=SWIFT_TEST_TIMEOUT) public void testDirectoriesOffRootHaveMatchingFileStatus() throws Exception {
  Path test=path("/test");
  fs.delete(test,true);
  mkdirs(test);
  assertExists("created test directory",test);
  FileStatus[] statuses=fs.listStatus(test);
  String statusString=statusToString(test.toString(),statuses);
  assertEquals("Wrong number of elements in file status " + statusString,0,statuses.length);
  Path src=path("/test/file");
  SwiftTestUtils.touch(fs,src);
  statuses=fs.listStatus(test);
  statusString=statusToString(test.toString(),statuses);
  assertEquals("Wrong number of elements in file status " + statusString,1,statuses.length);
  SwiftFileStatus stat=(SwiftFileStatus)statuses[0];
  assertTrue("isDir(): Not a directory: " + stat,stat.isDir());
  extraStatusAssertions(stat);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * test that a dir two levels down has a listStatus() call that
 * works as expected.
 * @throws Exception on failures
 */
@Test(timeout=SWIFT_TEST_TIMEOUT) public void testDirectoriesLowerDownHaveMatchingFileStatus() throws Exception {
  Path test=path("/test/testDirectoriesLowerDownHaveMatchingFileStatus");
  fs.delete(test,true);
  mkdirs(test);
  assertExists("created test sub directory",test);
  FileStatus[] statuses=fs.listStatus(test);
  String statusString=statusToString(test.toString(),statuses);
  assertEquals("Wrong number of elements in file status " + statusString,0,statuses.length);
}

Class: org.apache.hadoop.fs.swift.TestSwiftFileSystemExtendedContract

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testWriteReadFile() throws Exception {
  final Path f=new Path("/test/test");
  final FSDataOutputStream fsDataOutputStream=fs.create(f);
  final String message="Test string";
  fsDataOutputStream.write(message.getBytes());
  fsDataOutputStream.close();
  assertExists("created file",f);
  FSDataInputStream open=null;
  try {
    open=fs.open(f);
    final byte[] bytes=new byte[512];
    final int read=open.read(bytes);
    final byte[] buffer=new byte[read];
    System.arraycopy(bytes,0,buffer,0,read);
    assertEquals(message,new String(buffer));
  }
  finally {
    fs.delete(f,false);
    IOUtils.closeStream(open);
  }
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Assert that a filesystem is case sensitive.
 * This is done by creating a mixed-case filename and asserting that
 * its lower case version is not there.
 * @throws Exception failures
 */
@Test(timeout=SWIFT_TEST_TIMEOUT) public void testFilesystemIsCaseSensitive() throws Exception {
  String mixedCaseFilename="/test/UPPER.TXT";
  Path upper=path(mixedCaseFilename);
  Path lower=path(mixedCaseFilename.toLowerCase(Locale.ENGLISH));
  assertFalse("File exists" + upper,fs.exists(upper));
  assertFalse("File exists" + lower,fs.exists(lower));
  FSDataOutputStream out=fs.create(upper);
  out.writeUTF("UPPER");
  out.close();
  FileStatus upperStatus=fs.getFileStatus(upper);
  assertExists("Original upper case file" + upper,upper);
  assertPathDoesNotExist("lower case file",lower);
  out=fs.create(lower);
  out.writeUTF("l");
  out.close();
  assertExists("lower case file",lower);
  assertExists("Original upper case file " + upper,upper);
  FileStatus newStatus=fs.getFileStatus(upper);
  assertEquals("Expected status:" + upperStatus + " actual status "+ newStatus,upperStatus.getLen(),newStatus.getLen());
}

Class: org.apache.hadoop.fs.swift.TestSwiftFileSystemLsOperations

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testListLevelTestHadoop() throws Exception {
  createTestSubdirs();
  FileStatus[] paths;
  paths=fs.listStatus(path("/test/hadoop"));
  String stats=dumpStats("/test/hadoop",paths);
  assertEquals("Paths.length wrong in " + stats,3,paths.length);
  assertEquals("Path element[0] wrong: " + stats,path("/test/hadoop/a"),paths[0].getPath());
  assertEquals("Path element[1] wrong: " + stats,path("/test/hadoop/b"),paths[1].getPath());
  assertEquals("Path element[2] wrong: " + stats,path("/test/hadoop/c"),paths[2].getPath());
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testListNonEmptyRoot() throws Throwable {
  Path test=path("/test");
  touch(fs,test);
  FileStatus[] fileStatuses=fs.listStatus(path("/"));
  String stats=dumpStats("/",fileStatuses);
  assertEquals("Wrong #of root children" + stats,1,fileStatuses.length);
  FileStatus status=fileStatuses[0];
  assertEquals("Wrong path value" + stats,test,status.getPath());
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testListStatusFile() throws Exception {
  describe("Create a single file under /test;" + " assert that listStatus(/test) finds it");
  Path file=path("/test/filename");
  createFile(file);
  FileStatus[] pathStats=fs.listStatus(file);
  assertEquals(dumpStats("/test/",pathStats),1,pathStats.length);
  FileStatus lsStat=pathStats[0];
  assertEquals("Wrong file len in listing of " + lsStat,data.length,lsStat.getLen());
}

Class: org.apache.hadoop.fs.swift.TestSwiftFileSystemPartitionedUploads

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier HybridVerifier 

/** 
 * tests functionality for big files ( > 5Gb) upload
 */
@Test(timeout=SWIFT_BULK_IO_TEST_TIMEOUT) public void testFilePartUpload() throws Throwable {
  final Path path=new Path("/test/testFilePartUpload");
  int len=8192;
  final byte[] src=SwiftTestUtils.dataset(len,32,144);
  FSDataOutputStream out=fs.create(path,false,getBufferSize(),(short)1,BLOCK_SIZE);
  try {
    int totalPartitionsToWrite=len / PART_SIZE_BYTES;
    assertPartitionsWritten("Startup",out,0);
    int firstWriteLen=2048;
    out.write(src,0,firstWriteLen);
    long expected=getExpectedPartitionsWritten(firstWriteLen,PART_SIZE_BYTES,false);
    SwiftUtils.debug(LOG,"First write: predict %d partitions written",expected);
    assertPartitionsWritten("First write completed",out,expected);
    int remainder=len - firstWriteLen;
    SwiftUtils.debug(LOG,"remainder: writing: %d bytes",remainder);
    out.write(src,firstWriteLen,remainder);
    expected=getExpectedPartitionsWritten(len,PART_SIZE_BYTES,false);
    assertPartitionsWritten("Remaining data",out,expected);
    out.close();
    expected=getExpectedPartitionsWritten(len,PART_SIZE_BYTES,true);
    assertPartitionsWritten("Stream closed",out,expected);
    Header[] headers=fs.getStore().getObjectHeaders(path,true);
    for (    Header header : headers) {
      LOG.info(header.toString());
    }
    byte[] dest=readDataset(fs,path,len);
    LOG.info("Read dataset from " + path + ": data length ="+ len);
    SwiftTestUtils.compareByteArrays(src,dest,len);
    FileStatus status;
    final Path qualifiedPath=path.makeQualified(fs);
    status=fs.getFileStatus(qualifiedPath);
    BlockLocation[] locations=fs.getFileBlockLocations(status,0,len);
    assertNotNull("Null getFileBlockLocations()",locations);
    assertTrue("empty array returned for getFileBlockLocations()",locations.length > 0);
    try {
      validatePathLen(path,len);
    }
 catch (    AssertionError e) {
      throw new AssumptionViolatedException(e,null);
    }
  }
  finally {
    IOUtils.closeStream(out);
  }
}

APIUtilityVerifier EqualityVerifier 

/** 
 * Test that when a partitioned file is overwritten by a smaller one,
 * all the old partitioned files go away
 * @throws Throwable
 */
@Test(timeout=SWIFT_BULK_IO_TEST_TIMEOUT) public void testOverwritePartitionedFile() throws Throwable {
  final Path path=new Path("/test/testOverwritePartitionedFile");
  final int len1=8192;
  final byte[] src1=SwiftTestUtils.dataset(len1,'A','Z');
  FSDataOutputStream out=fs.create(path,false,getBufferSize(),(short)1,1024);
  out.write(src1,0,len1);
  out.close();
  long expected=getExpectedPartitionsWritten(len1,PART_SIZE_BYTES,false);
  assertPartitionsWritten("initial upload",out,expected);
  assertExists("Exists",path);
  FileStatus status=fs.getFileStatus(path);
  assertEquals("Length",len1,status.getLen());
  final int len2=4095;
  final byte[] src2=SwiftTestUtils.dataset(len2,'a','z');
  out=fs.create(path,true,getBufferSize(),(short)1,1024);
  out.write(src2,0,len2);
  out.close();
  status=fs.getFileStatus(path);
  assertEquals("Length",len2,status.getLen());
  byte[] dest=readDataset(fs,path,len2);
  SwiftTestUtils.compareByteArrays(src2,dest,len2);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test sticks up a very large partitioned file and verifies that
 * it comes back unchanged.
 * @throws Throwable
 */
@Test(timeout=SWIFT_BULK_IO_TEST_TIMEOUT) public void testManyPartitionedFile() throws Throwable {
  final Path path=new Path("/test/testManyPartitionedFile");
  int len=PART_SIZE_BYTES * 15;
  final byte[] src=SwiftTestUtils.dataset(len,32,144);
  FSDataOutputStream out=fs.create(path,false,getBufferSize(),(short)1,BLOCK_SIZE);
  out.write(src,0,src.length);
  int expected=getExpectedPartitionsWritten(len,PART_SIZE_BYTES,true);
  out.close();
  assertPartitionsWritten("write completed",out,expected);
  assertEquals("too few bytes written",len,SwiftNativeFileSystem.getBytesWritten(out));
  assertEquals("too few bytes uploaded",len,SwiftNativeFileSystem.getBytesUploaded(out));
  byte[] dest=readDataset(fs,path,len);
  SwiftTestUtils.compareByteArrays(src,dest,len);
  FileStatus[] stats=fs.listStatus(path);
  assertEquals("wrong entry count in " + SwiftTestUtils.dumpStats(path.toString(),stats),expected,stats.length);
}

APIUtilityVerifier BooleanVerifier NullVerifier HybridVerifier 

/** 
 * tests functionality for big files ( > 5Gb) upload
 */
@Test(timeout=SWIFT_BULK_IO_TEST_TIMEOUT) public void testFilePartUploadNoLengthCheck() throws IOException, URISyntaxException {
  final Path path=new Path("/test/testFilePartUploadLengthCheck");
  int len=8192;
  final byte[] src=SwiftTestUtils.dataset(len,32,144);
  FSDataOutputStream out=fs.create(path,false,getBufferSize(),(short)1,BLOCK_SIZE);
  try {
    int totalPartitionsToWrite=len / PART_SIZE_BYTES;
    assertPartitionsWritten("Startup",out,0);
    int firstWriteLen=2048;
    out.write(src,0,firstWriteLen);
    long expected=getExpectedPartitionsWritten(firstWriteLen,PART_SIZE_BYTES,false);
    SwiftUtils.debug(LOG,"First write: predict %d partitions written",expected);
    assertPartitionsWritten("First write completed",out,expected);
    int remainder=len - firstWriteLen;
    SwiftUtils.debug(LOG,"remainder: writing: %d bytes",remainder);
    out.write(src,firstWriteLen,remainder);
    expected=getExpectedPartitionsWritten(len,PART_SIZE_BYTES,false);
    assertPartitionsWritten("Remaining data",out,expected);
    out.close();
    expected=getExpectedPartitionsWritten(len,PART_SIZE_BYTES,true);
    assertPartitionsWritten("Stream closed",out,expected);
    Header[] headers=fs.getStore().getObjectHeaders(path,true);
    for (    Header header : headers) {
      LOG.info(header.toString());
    }
    byte[] dest=readDataset(fs,path,len);
    LOG.info("Read dataset from " + path + ": data length ="+ len);
    SwiftTestUtils.compareByteArrays(src,dest,len);
    FileStatus status=fs.getFileStatus(path);
    BlockLocation[] locations=fs.getFileBlockLocations(status,0,len);
    assertNotNull("Null getFileBlockLocations()",locations);
    assertTrue("empty array returned for getFileBlockLocations()",locations.length > 0);
  }
  finally {
    IOUtils.closeStream(out);
  }
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_BULK_IO_TEST_TIMEOUT) public void testRenamePartitionedFile() throws Throwable {
  Path src=new Path("/test/testRenamePartitionedFileSrc");
  int len=data.length;
  SwiftTestUtils.writeDataset(fs,src,data,len,1024,false);
  assertExists("Exists",src);
  String partOneName=SwiftUtils.partitionFilenameFromNumber(1);
  Path srcPart=new Path(src,partOneName);
  Path dest=new Path("/test/testRenamePartitionedFileDest");
  Path destPart=new Path(src,partOneName);
  assertExists("Partition Exists",srcPart);
  fs.rename(src,dest);
  assertPathExists(fs,"dest file missing",dest);
  FileStatus status=fs.getFileStatus(dest);
  assertEquals("Length of renamed file is wrong",len,status.getLen());
  byte[] destData=readDataset(fs,dest,len);
  SwiftTestUtils.compareByteArrays(data,destData,len);
  String srcLs=SwiftTestUtils.ls(fs,src);
  String destLs=SwiftTestUtils.ls(fs,dest);
  assertPathDoesNotExist("deleted file still found in " + srcLs,src);
  assertPathDoesNotExist("partition file still found in " + srcLs,srcPart);
}

Class: org.apache.hadoop.fs.swift.TestSwiftFileSystemRead

APIUtilityVerifier EqualityVerifier 

/** 
 * Read and write some JSON
 * @throws IOException
 */
@Test(timeout=SWIFT_TEST_TIMEOUT) public void testRWJson() throws IOException {
  final String message="{" + " 'json': { 'i':43, 'b':true}," + " 's':'string'"+ "}";
  final Path filePath=new Path("/test/file.json");
  writeTextFile(fs,filePath,message,false);
  String readJson=readBytesToString(fs,filePath,message.length());
  assertEquals(message,readJson);
  FileStatus status=fs.getFileStatus(filePath);
  BlockLocation[] locations=fs.getFileBlockLocations(status,0,10);
}

APIUtilityVerifier EqualityVerifier 

/** 
 * Read and write some XML
 * @throws IOException
 */
@Test(timeout=SWIFT_TEST_TIMEOUT) public void testRWXML() throws IOException {
  final String message="" + " " + " string"+ "";
  final Path filePath=new Path("/test/file.xml");
  writeTextFile(fs,filePath,message,false);
  String read=readBytesToString(fs,filePath,message.length());
  assertEquals(message,read);
}

Class: org.apache.hadoop.fs.swift.TestSwiftFileSystemRename

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testRenameFile() throws Exception {
  assumeRenameSupported();
  final Path old=new Path("/test/alice/file");
  final Path newPath=new Path("/test/bob/file");
  fs.mkdirs(newPath.getParent());
  final FSDataOutputStream fsDataOutputStream=fs.create(old);
  final byte[] message="Some data".getBytes();
  fsDataOutputStream.write(message);
  fsDataOutputStream.close();
  assertTrue(fs.exists(old));
  rename(old,newPath,true,false,true);
  final FSDataInputStream bobStream=fs.open(newPath);
  final byte[] bytes=new byte[512];
  final int read=bobStream.read(bytes);
  bobStream.close();
  final byte[] buffer=new byte[read];
  System.arraycopy(bytes,0,buffer,0,read);
  assertEquals(new String(message),new String(buffer));
}

APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier HybridVerifier 

/** 
 * Rename a file into a directory
 * @throws Exception
 */
@Test(timeout=SWIFT_TEST_TIMEOUT) public void testRenameFileIntoExistingDirectory() throws Exception {
  assumeRenameSupported();
  Path src=path("/test/olddir/file");
  createFile(src);
  Path dst=path("/test/new/newdir");
  fs.mkdirs(dst);
  rename(src,dst,true,false,true);
  Path newFile=path("/test/new/newdir/file");
  if (!fs.exists(newFile)) {
    String ls=ls(dst);
    LOG.info(ls(path("/test/new")));
    LOG.info(ls(path("/test/hadoop")));
    fail("did not find " + newFile + " - directory: "+ ls);
  }
  assertTrue("Destination changed",fs.exists(path("/test/new/newdir/file")));
}

Class: org.apache.hadoop.fs.swift.TestSwiftObjectPath

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testParseUrlPath() throws Exception {
  final String pathString="swift://container.service1/home/user/files/file1";
  final URI uri=new URI(pathString);
  final Path path=new Path(pathString);
  final SwiftObjectPath expected=SwiftObjectPath.fromPath(uri,path);
  final SwiftObjectPath actual=new SwiftObjectPath(RestClientBindings.extractContainerName(uri),"/home/user/files/file1");
  assertEquals(expected,actual);
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testParsePath() throws Exception {
  final String pathString="/home/user/files/file1";
  final Path path=new Path(pathString);
  final URI uri=new URI("http://container.localhost");
  final SwiftObjectPath expected=SwiftObjectPath.fromPath(uri,path);
  final SwiftObjectPath actual=new SwiftObjectPath(RestClientBindings.extractContainerName(uri),pathString);
  assertEquals(expected,actual);
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testParseAuthenticatedUrl() throws Exception {
  final String pathString="swift://container.service1/v2/AUTH_00345h34l93459y4/home/tom/documents/finance.docx";
  final URI uri=new URI(pathString);
  final Path path=new Path(pathString);
  final SwiftObjectPath expected=SwiftObjectPath.fromPath(uri,path);
  final SwiftObjectPath actual=new SwiftObjectPath(RestClientBindings.extractContainerName(uri),"/home/tom/documents/finance.docx");
  assertEquals(expected,actual);
}

Class: org.apache.hadoop.fs.swift.http.TestSwiftRestClient

APIUtilityVerifier UtilityVerifier 

@Test(timeout=SWIFT_TEST_TIMEOUT) public void testPutAndDelete() throws Throwable {
  assumeEnabled();
  SwiftRestClient client=createClient();
  client.authenticate();
  Path path=new Path("restTestPutAndDelete");
  SwiftObjectPath sobject=SwiftObjectPath.fromPath(serviceURI,path);
  byte[] stuff=new byte[1];
  stuff[0]='a';
  client.upload(sobject,new ByteArrayInputStream(stuff),stuff.length);
  Duration head=new Duration();
  Header[] responseHeaders=client.headRequest("expect success",sobject,SwiftRestClient.NEWEST);
  head.finished();
  LOG.info("head request duration " + head);
  for (  Header header : responseHeaders) {
    LOG.info(header.toString());
  }
  client.delete(sobject);
  try {
    Header[] headers=client.headRequest("expect fail",sobject,SwiftRestClient.NEWEST);
    Assert.fail("Expected deleted file, but object is still present: " + sobject);
  }
 catch (  FileNotFoundException e) {
  }
  for (  DurationStats stats : client.getOperationStatistics()) {
    LOG.info(stats);
  }
}

Class: org.apache.hadoop.fs.swift.scale.TestWriteManySmallFiles

APIUtilityVerifier IterativeVerifier EqualityVerifier 

@Test(timeout=SWIFT_BULK_IO_TEST_TIMEOUT) public void testScaledWriteThenRead() throws Throwable {
  Path dir=new Path("/test/manysmallfiles");
  Duration rm1=new Duration();
  fs.delete(dir,true);
  rm1.finished();
  fs.mkdirs(dir);
  Duration ls1=new Duration();
  fs.listStatus(dir);
  ls1.finished();
  long count=getOperationCount();
  SwiftTestUtils.noteAction("Beginning Write of " + count + " files ");
  DurationStats writeStats=new DurationStats("write");
  DurationStats readStats=new DurationStats("read");
  String format="%08d";
  for (long l=0; l < count; l++) {
    String name=String.format(format,l);
    Path p=new Path(dir,"part-" + name);
    Duration d=new Duration();
    SwiftTestUtils.writeTextFile(fs,p,name,false);
    d.finished();
    writeStats.add(d);
    Thread.sleep(1000);
  }
  SwiftTestUtils.noteAction("Beginning ls");
  Duration ls2=new Duration();
  FileStatus[] status2=(FileStatus[])fs.listStatus(dir);
  ls2.finished();
  assertEquals("Not enough entries in the directory",count,status2.length);
  SwiftTestUtils.noteAction("Beginning read");
  for (long l=0; l < count; l++) {
    String name=String.format(format,l);
    Path p=new Path(dir,"part-" + name);
    Duration d=new Duration();
    String result=SwiftTestUtils.readBytesToString(fs,p,name.length());
    assertEquals(name,result);
    d.finished();
    readStats.add(d);
  }
  SwiftTestUtils.noteAction("Beginning delete");
  Duration rm2=new Duration();
  fs.delete(dir,true);
  rm2.finished();
  LOG.info(String.format("'filesystem','%s'",fs.getUri()));
  LOG.info(writeStats.toString());
  LOG.info(readStats.toString());
  LOG.info(String.format("'rm1',%d,'ls1',%d",rm1.value(),ls1.value()));
  LOG.info(String.format("'rm2',%d,'ls2',%d",rm2.value(),ls2.value()));
}

Class: org.apache.hadoop.fs.viewfs.TestChRootedFileSystem

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testList() throws IOException {
  FileStatus fs=fSys.getFileStatus(new Path("/"));
  Assert.assertTrue(fs.isDirectory());
  Assert.assertEquals(fs.getPath(),chrootedTo);
  FileStatus[] dirPaths=fSys.listStatus(new Path("/"));
  Assert.assertEquals(0,dirPaths.length);
  fileSystemTestHelper.createFile(fSys,"/foo");
  fileSystemTestHelper.createFile(fSys,"/bar");
  fSys.mkdirs(new Path("/dirX"));
  fSys.mkdirs(fileSystemTestHelper.getTestRootPath(fSys,"/dirY"));
  fSys.mkdirs(new Path("/dirX/dirXX"));
  dirPaths=fSys.listStatus(new Path("/"));
  Assert.assertEquals(4,dirPaths.length);
  fs=FileSystemTestHelper.containsPath(new Path(chrootedTo,"foo"),dirPaths);
  Assert.assertNotNull(fs);
  Assert.assertTrue(fs.isFile());
  fs=FileSystemTestHelper.containsPath(new Path(chrootedTo,"bar"),dirPaths);
  Assert.assertNotNull(fs);
  Assert.assertTrue(fs.isFile());
  fs=FileSystemTestHelper.containsPath(new Path(chrootedTo,"dirX"),dirPaths);
  Assert.assertNotNull(fs);
  Assert.assertTrue(fs.isDirectory());
  fs=FileSystemTestHelper.containsPath(new Path(chrootedTo,"dirY"),dirPaths);
  Assert.assertNotNull(fs);
  Assert.assertTrue(fs.isDirectory());
}

Class: org.apache.hadoop.fs.viewfs.TestViewFileSystemDelegationTokenSupport

APIUtilityVerifier InternalCallVerifier NullVerifier 

/** 
 * Regression test for HADOOP-8408.
 */
@Test public void testGetCanonicalServiceNameWithNonDefaultMountTable() throws URISyntaxException, IOException {
  Configuration conf=new Configuration();
  ConfigUtil.addLink(conf,MOUNT_TABLE_NAME,"/user",new URI("file:///"));
  FileSystem viewFs=FileSystem.get(new URI(FsConstants.VIEWFS_SCHEME + "://" + MOUNT_TABLE_NAME),conf);
  String serviceName=viewFs.getCanonicalServiceName();
  assertNull(serviceName);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testAddDelegationTokens() throws Exception {
  Credentials creds=new Credentials();
  Token fs1Tokens[]=addTokensWithCreds(fs1,creds);
  assertEquals(1,fs1Tokens.length);
  assertEquals(1,creds.numberOfTokens());
  Token fs2Tokens[]=addTokensWithCreds(fs2,creds);
  assertEquals(1,fs2Tokens.length);
  assertEquals(2,creds.numberOfTokens());
  Credentials savedCreds=creds;
  creds=new Credentials();
  Token viewFsTokens[]=viewFs.addDelegationTokens("me",creds);
  assertEquals(2,viewFsTokens.length);
  assertTrue(creds.getAllTokens().containsAll(savedCreds.getAllTokens()));
  assertEquals(savedCreds.numberOfTokens(),creds.numberOfTokens());
  viewFsTokens=viewFs.addDelegationTokens("me",creds);
  assertEquals(0,viewFsTokens.length);
  assertTrue(creds.getAllTokens().containsAll(savedCreds.getAllTokens()));
  assertEquals(savedCreds.numberOfTokens(),creds.numberOfTokens());
}

APIUtilityVerifier InternalCallVerifier NullVerifier 

@Test public void testGetCanonicalServiceNameWithDefaultMountTable() throws URISyntaxException, IOException {
  Configuration conf=new Configuration();
  ConfigUtil.addLink(conf,"/user",new URI("file:///"));
  FileSystem viewFs=FileSystem.get(FsConstants.VIEWFS_URI,conf);
  String serviceName=viewFs.getCanonicalServiceName();
  assertNull(serviceName);
}

Class: org.apache.hadoop.fs.viewfs.TestViewfsFileStatus

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testFileStatusSerialziation() throws IOException, URISyntaxException {
  String testfilename="testFileStatusSerialziation";
  TEST_DIR.mkdirs();
  File infile=new File(TEST_DIR,testfilename);
  final byte[] content="dingos".getBytes();
  FileOutputStream fos=null;
  try {
    fos=new FileOutputStream(infile);
    fos.write(content);
  }
  finally {
    if (fos != null) {
      fos.close();
    }
  }
  assertEquals((long)content.length,infile.length());
  Configuration conf=new Configuration();
  ConfigUtil.addLink(conf,"/foo/bar/baz",TEST_DIR.toURI());
  FileSystem vfs=FileSystem.get(FsConstants.VIEWFS_URI,conf);
  assertEquals(ViewFileSystem.class,vfs.getClass());
  FileStatus stat=vfs.getFileStatus(new Path("/foo/bar/baz",testfilename));
  assertEquals(content.length,stat.getLen());
  DataOutputBuffer dob=new DataOutputBuffer();
  stat.write(dob);
  DataInputBuffer dib=new DataInputBuffer();
  dib.reset(dob.getData(),0,dob.getLength());
  FileStatus deSer=new FileStatus();
  deSer.readFields(dib);
  assertEquals(content.length,deSer.getLen());
}

Class: org.apache.hadoop.ha.TestZKFailoverController

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test that automatic failover won't run against a target that hasn't
 * explicitly enabled the feature.
 */
@Test(timeout=10000) public void testWontRunWhenAutoFailoverDisabled() throws Exception {
  DummyHAService svc=cluster.getService(1);
  svc=Mockito.spy(svc);
  Mockito.doReturn(false).when(svc).isAutoFailoverEnabled();
  assertEquals(ZKFailoverController.ERR_CODE_AUTO_FAILOVER_NOT_ENABLED,runFC(svc,"-formatZK"));
  assertEquals(ZKFailoverController.ERR_CODE_AUTO_FAILOVER_NOT_ENABLED,runFC(svc));
}

Class: org.apache.hadoop.hdfs.TestAbandonBlock

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testAbandonBlock() throws IOException {
  String src=FILE_NAME_PREFIX + "foo";
  FSDataOutputStream fout=fs.create(new Path(src),true,4096,(short)1,512L);
  for (int i=0; i < 1024; i++) {
    fout.write(123);
  }
  fout.hflush();
  long fileId=((DFSOutputStream)fout.getWrappedStream()).getFileId();
  DFSClient dfsclient=DFSClientAdapter.getDFSClient(fs);
  LocatedBlocks blocks=dfsclient.getNamenode().getBlockLocations(src,0,Integer.MAX_VALUE);
  int orginalNumBlocks=blocks.locatedBlockCount();
  LocatedBlock b=blocks.getLastLocatedBlock();
  dfsclient.getNamenode().abandonBlock(b.getBlock(),fileId,src,dfsclient.clientName);
  dfsclient.getNamenode().abandonBlock(b.getBlock(),fileId,src,dfsclient.clientName);
  fout.close();
  cluster.restartNameNode();
  blocks=dfsclient.getNamenode().getBlockLocations(src,0,Integer.MAX_VALUE);
  Assert.assertEquals("Blocks " + b + " has not been abandoned.",orginalNumBlocks,blocks.locatedBlockCount() + 1);
}

Class: org.apache.hadoop.hdfs.TestBlockReaderFactory

APIUtilityVerifier InternalCallVerifier BooleanVerifier PublicFieldVerifier 

/** 
 * Test shutting down the ShortCircuitCache while there are things in it.
 */
@Test public void testShortCircuitCacheShutdown() throws Exception {
  TemporarySocketDirectory sockDir=new TemporarySocketDirectory();
  Configuration conf=createShortCircuitConf("testShortCircuitCacheShutdown",sockDir);
  conf.set(DFS_CLIENT_CONTEXT,"testShortCircuitCacheShutdown");
  Configuration serverConf=new Configuration(conf);
  DFSInputStream.tcpReadsDisabledForTesting=true;
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
  cluster.waitActive();
  final DistributedFileSystem fs=(DistributedFileSystem)FileSystem.get(cluster.getURI(0),conf);
  final String TEST_FILE="/test_file";
  final int TEST_FILE_LEN=4000;
  final int SEED=0xFADEC;
  DFSTestUtil.createFile(fs,new Path(TEST_FILE),TEST_FILE_LEN,(short)1,SEED);
  byte contents[]=DFSTestUtil.readFileBuffer(fs,new Path(TEST_FILE));
  byte expected[]=DFSTestUtil.calculateFileContentsFromSeed(SEED,TEST_FILE_LEN);
  Assert.assertTrue(Arrays.equals(contents,expected));
  final ShortCircuitCache cache=fs.dfs.getClientContext().getShortCircuitCache();
  cache.close();
  Assert.assertTrue(cache.getDfsClientShmManager().getDomainSocketWatcher().isClosed());
  cluster.shutdown();
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * When an InterruptedException is sent to a thread calling
 * FileChannel#read, the FileChannel is immediately closed and the
 * thread gets an exception.  This effectively means that we might have
 * someone asynchronously calling close() on the file descriptors we use
 * in BlockReaderLocal.  So when unreferencing a ShortCircuitReplica in
 * ShortCircuitCache#unref, we should check if the FileChannel objects
 * are still open.  If not, we should purge the replica to avoid giving
 * it out to any future readers.
 * This is a regression test for HDFS-6227: Short circuit read failed
 * due to ClosedChannelException.
 * Note that you may still get ClosedChannelException errors if two threads
 * are reading from the same replica and an InterruptedException is delivered
 * to one of them.
 */
@Test(timeout=120000) public void testPurgingClosedReplicas() throws Exception {
  BlockReaderTestUtil.enableBlockReaderFactoryTracing();
  final AtomicInteger replicasCreated=new AtomicInteger(0);
  final AtomicBoolean testFailed=new AtomicBoolean(false);
  DFSInputStream.tcpReadsDisabledForTesting=true;
  BlockReaderFactory.createShortCircuitReplicaInfoCallback=new ShortCircuitCache.ShortCircuitReplicaCreator(){
    @Override public ShortCircuitReplicaInfo createShortCircuitReplicaInfo(){
      replicasCreated.incrementAndGet();
      return null;
    }
  }
;
  TemporarySocketDirectory sockDir=new TemporarySocketDirectory();
  Configuration conf=createShortCircuitConf("testPurgingClosedReplicas",sockDir);
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  final DistributedFileSystem dfs=cluster.getFileSystem();
  final String TEST_FILE="/test_file";
  final int TEST_FILE_LEN=4095;
  final int SEED=0xFADE0;
  final DistributedFileSystem fs=(DistributedFileSystem)FileSystem.get(cluster.getURI(0),conf);
  DFSTestUtil.createFile(fs,new Path(TEST_FILE),TEST_FILE_LEN,(short)1,SEED);
  final Semaphore sem=new Semaphore(0);
  final List locatedBlocks=cluster.getNameNode().getRpcServer().getBlockLocations(TEST_FILE,0,TEST_FILE_LEN).getLocatedBlocks();
  final LocatedBlock lblock=locatedBlocks.get(0);
  final byte[] buf=new byte[TEST_FILE_LEN];
  Runnable readerRunnable=new Runnable(){
    @Override public void run(){
      try {
        while (true) {
          BlockReader blockReader=null;
          try {
            blockReader=BlockReaderTestUtil.getBlockReader(cluster,lblock,0,TEST_FILE_LEN);
            sem.release();
            try {
              blockReader.readAll(buf,0,TEST_FILE_LEN);
            }
  finally {
              sem.acquireUninterruptibly();
            }
          }
 catch (          ClosedByInterruptException e) {
            LOG.info("got the expected ClosedByInterruptException",e);
            sem.release();
            break;
          }
 finally {
            if (blockReader != null)             blockReader.close();
          }
          LOG.info("read another " + TEST_FILE_LEN + " bytes.");
        }
      }
 catch (      Throwable t) {
        LOG.error("getBlockReader failure",t);
        testFailed.set(true);
        sem.release();
      }
    }
  }
;
  Thread thread=new Thread(readerRunnable);
  thread.start();
  while (thread.isAlive()) {
    sem.acquireUninterruptibly();
    thread.interrupt();
    sem.release();
  }
  Assert.assertFalse(testFailed.get());
  BlockReader blockReader=null;
  try {
    blockReader=BlockReaderTestUtil.getBlockReader(cluster,lblock,0,TEST_FILE_LEN);
    blockReader.readFully(buf,0,TEST_FILE_LEN);
  }
  finally {
    if (blockReader != null)     blockReader.close();
  }
  byte expected[]=DFSTestUtil.calculateFileContentsFromSeed(SEED,TEST_FILE_LEN);
  Assert.assertTrue(Arrays.equals(buf,expected));
  Assert.assertEquals(2,replicasCreated.get());
  dfs.close();
  cluster.shutdown();
  sockDir.close();
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * If we have a UNIX domain socket configured,
 * and we have dfs.client.domain.socket.data.traffic set to true,
 * and short-circuit access fails, we should still be able to pass
 * data traffic over the UNIX domain socket.  Test this.
 */
@Test(timeout=60000) public void testFallbackFromShortCircuitToUnixDomainTraffic() throws Exception {
  DFSInputStream.tcpReadsDisabledForTesting=true;
  TemporarySocketDirectory sockDir=new TemporarySocketDirectory();
  Configuration clientConf=createShortCircuitConf("testFallbackFromShortCircuitToUnixDomainTraffic",sockDir);
  clientConf.set(DFS_CLIENT_CONTEXT,"testFallbackFromShortCircuitToUnixDomainTraffic_clientContext");
  clientConf.setBoolean(DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC,true);
  Configuration serverConf=new Configuration(clientConf);
  serverConf.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_KEY,false);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
  cluster.waitActive();
  FileSystem dfs=FileSystem.get(cluster.getURI(0),clientConf);
  String TEST_FILE="/test_file";
  final int TEST_FILE_LEN=8193;
  final int SEED=0xFADED;
  DFSTestUtil.createFile(dfs,new Path(TEST_FILE),TEST_FILE_LEN,(short)1,SEED);
  byte contents[]=DFSTestUtil.readFileBuffer(dfs,new Path(TEST_FILE));
  byte expected[]=DFSTestUtil.calculateFileContentsFromSeed(SEED,TEST_FILE_LEN);
  Assert.assertTrue(Arrays.equals(contents,expected));
  cluster.shutdown();
  sockDir.close();
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that a client which supports short-circuit reads using
 * shared memory can fall back to not using shared memory when
 * the server doesn't support it.
 */
@Test public void testShortCircuitReadFromServerWithoutShm() throws Exception {
  TemporarySocketDirectory sockDir=new TemporarySocketDirectory();
  Configuration clientConf=createShortCircuitConf("testShortCircuitReadFromServerWithoutShm",sockDir);
  Configuration serverConf=new Configuration(clientConf);
  serverConf.setInt(DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS,0);
  DFSInputStream.tcpReadsDisabledForTesting=true;
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
  cluster.waitActive();
  clientConf.set(DFS_CLIENT_CONTEXT,"testShortCircuitReadFromServerWithoutShm_clientContext");
  final DistributedFileSystem fs=(DistributedFileSystem)FileSystem.get(cluster.getURI(0),clientConf);
  final String TEST_FILE="/test_file";
  final int TEST_FILE_LEN=4000;
  final int SEED=0xFADEC;
  DFSTestUtil.createFile(fs,new Path(TEST_FILE),TEST_FILE_LEN,(short)1,SEED);
  byte contents[]=DFSTestUtil.readFileBuffer(fs,new Path(TEST_FILE));
  byte expected[]=DFSTestUtil.calculateFileContentsFromSeed(SEED,TEST_FILE_LEN);
  Assert.assertTrue(Arrays.equals(contents,expected));
  final ShortCircuitCache cache=fs.dfs.getClientContext().getShortCircuitCache();
  final DatanodeInfo datanode=new DatanodeInfo(cluster.getDataNodes().get(0).getDatanodeId());
  cache.getDfsClientShmManager().visit(new Visitor(){
    @Override public void visit(    HashMap info) throws IOException {
      Assert.assertEquals(1,info.size());
      PerDatanodeVisitorInfo vinfo=info.get(datanode);
      Assert.assertTrue(vinfo.disabled);
      Assert.assertEquals(0,vinfo.full.size());
      Assert.assertEquals(0,vinfo.notFull.size());
    }
  }
);
  cluster.shutdown();
}

APIUtilityVerifier BranchVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * Test the case where we have multiple threads waiting on the
 * ShortCircuitCache delivering a certain ShortCircuitReplica.
 * In this case, there should only be one call to
 * createShortCircuitReplicaInfo.  This one replica should be shared
 * by all threads.
 */
@Test(timeout=60000) public void testMultipleWaitersOnShortCircuitCache() throws Exception {
  final CountDownLatch latch=new CountDownLatch(1);
  final AtomicBoolean creationIsBlocked=new AtomicBoolean(true);
  final AtomicBoolean testFailed=new AtomicBoolean(false);
  DFSInputStream.tcpReadsDisabledForTesting=true;
  BlockReaderFactory.createShortCircuitReplicaInfoCallback=new ShortCircuitCache.ShortCircuitReplicaCreator(){
    @Override public ShortCircuitReplicaInfo createShortCircuitReplicaInfo(){
      Uninterruptibles.awaitUninterruptibly(latch);
      if (!creationIsBlocked.compareAndSet(true,false)) {
        Assert.fail("there were multiple calls to " + "createShortCircuitReplicaInfo.  Only one was expected.");
      }
      return null;
    }
  }
;
  TemporarySocketDirectory sockDir=new TemporarySocketDirectory();
  Configuration conf=createShortCircuitConf("testMultipleWaitersOnShortCircuitCache",sockDir);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  final DistributedFileSystem dfs=cluster.getFileSystem();
  final String TEST_FILE="/test_file";
  final int TEST_FILE_LEN=4000;
  final int SEED=0xFADED;
  final int NUM_THREADS=10;
  DFSTestUtil.createFile(dfs,new Path(TEST_FILE),TEST_FILE_LEN,(short)1,SEED);
  Runnable readerRunnable=new Runnable(){
    @Override public void run(){
      try {
        byte contents[]=DFSTestUtil.readFileBuffer(dfs,new Path(TEST_FILE));
        Assert.assertFalse(creationIsBlocked.get());
        byte expected[]=DFSTestUtil.calculateFileContentsFromSeed(SEED,TEST_FILE_LEN);
        Assert.assertTrue(Arrays.equals(contents,expected));
      }
 catch (      Throwable e) {
        LOG.error("readerRunnable error",e);
        testFailed.set(true);
      }
    }
  }
;
  Thread threads[]=new Thread[NUM_THREADS];
  for (int i=0; i < NUM_THREADS; i++) {
    threads[i]=new Thread(readerRunnable);
    threads[i].start();
  }
  Thread.sleep(500);
  latch.countDown();
  for (int i=0; i < NUM_THREADS; i++) {
    Uninterruptibles.joinUninterruptibly(threads[i]);
  }
  cluster.shutdown();
  sockDir.close();
  Assert.assertFalse(testFailed.get());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Test that a client which does not support short-circuit reads using
 * shared memory can talk with a server which supports it.
 */
@Test public void testShortCircuitReadFromClientWithoutShm() throws Exception {
  TemporarySocketDirectory sockDir=new TemporarySocketDirectory();
  Configuration clientConf=createShortCircuitConf("testShortCircuitReadWithoutShm",sockDir);
  Configuration serverConf=new Configuration(clientConf);
  DFSInputStream.tcpReadsDisabledForTesting=true;
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build();
  cluster.waitActive();
  clientConf.setInt(DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS,0);
  clientConf.set(DFS_CLIENT_CONTEXT,"testShortCircuitReadFromClientWithoutShm_clientContext");
  final DistributedFileSystem fs=(DistributedFileSystem)FileSystem.get(cluster.getURI(0),clientConf);
  final String TEST_FILE="/test_file";
  final int TEST_FILE_LEN=4000;
  final int SEED=0xFADEC;
  DFSTestUtil.createFile(fs,new Path(TEST_FILE),TEST_FILE_LEN,(short)1,SEED);
  byte contents[]=DFSTestUtil.readFileBuffer(fs,new Path(TEST_FILE));
  byte expected[]=DFSTestUtil.calculateFileContentsFromSeed(SEED,TEST_FILE_LEN);
  Assert.assertTrue(Arrays.equals(contents,expected));
  final ShortCircuitCache cache=fs.dfs.getClientContext().getShortCircuitCache();
  Assert.assertEquals(null,cache.getDfsClientShmManager());
  cluster.shutdown();
}

Class: org.apache.hadoop.hdfs.TestBlockReaderLocalLegacy

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that, in the case of an error, the position and limit of a ByteBuffer
 * are left unchanged. This is not mandated by ByteBufferReadable, but clients
 * of this class might immediately issue a retry on failure, so it's polite.
 */
@Test public void testStablePositionAfterCorruptRead() throws Exception {
  final short REPL_FACTOR=1;
  final long FILE_LENGTH=512L;
  HdfsConfiguration conf=getConfiguration(null);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  FileSystem fs=cluster.getFileSystem();
  Path path=new Path("/corrupted");
  DFSTestUtil.createFile(fs,path,FILE_LENGTH,REPL_FACTOR,12345L);
  DFSTestUtil.waitReplication(fs,path,REPL_FACTOR);
  ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,path);
  int blockFilesCorrupted=cluster.corruptBlockOnDataNodes(block);
  assertEquals("All replicas not corrupted",REPL_FACTOR,blockFilesCorrupted);
  FSDataInputStream dis=cluster.getFileSystem().open(path);
  ByteBuffer buf=ByteBuffer.allocateDirect((int)FILE_LENGTH);
  boolean sawException=false;
  try {
    dis.read(buf);
  }
 catch (  ChecksumException ex) {
    sawException=true;
  }
  assertTrue(sawException);
  assertEquals(0,buf.position());
  assertEquals(buf.capacity(),buf.limit());
  dis=cluster.getFileSystem().open(path);
  buf.position(3);
  buf.limit(25);
  sawException=false;
  try {
    dis.read(buf);
  }
 catch (  ChecksumException ex) {
    sawException=true;
  }
  assertTrue(sawException);
  assertEquals(3,buf.position());
  assertEquals(25,buf.limit());
  cluster.shutdown();
}

Class: org.apache.hadoop.hdfs.TestBlocksScheduledCounter

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testBlocksScheduledCounter() throws IOException {
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(new HdfsConfiguration()).build();
  cluster.waitActive();
  FileSystem fs=cluster.getFileSystem();
  FSDataOutputStream out=fs.create(new Path("/testBlockScheduledCounter"));
  for (int i=0; i < 1024; i++) {
    out.write(i);
  }
  out.hflush();
  ArrayList dnList=new ArrayList();
  final DatanodeManager dm=cluster.getNamesystem().getBlockManager().getDatanodeManager();
  dm.fetchDatanodes(dnList,dnList,false);
  DatanodeDescriptor dn=dnList.get(0);
  assertEquals(1,dn.getBlocksScheduled());
  out.close();
  assertEquals(0,dn.getBlocksScheduled());
}

Class: org.apache.hadoop.hdfs.TestDFSAddressConfig

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier NullVerifier HybridVerifier 

@Test public void testDFSAddressConfig() throws IOException {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  cluster.waitActive();
  ArrayList dns=cluster.getDataNodes();
  DataNode dn=dns.get(0);
  String selfSocketAddr=dn.getXferAddress().toString();
  System.out.println("DN Self Socket Addr == " + selfSocketAddr);
  assertTrue(selfSocketAddr.contains("/127.0.0.1:"));
  for (int i=0; i < dns.size(); i++) {
    DataNodeProperties dnp=cluster.stopDataNode(i);
    assertNotNull("Should have been able to stop simulated datanode",dnp);
  }
  conf.unset(DFS_DATANODE_ADDRESS_KEY);
  conf.unset(DFS_DATANODE_HTTP_ADDRESS_KEY);
  conf.unset(DFS_DATANODE_IPC_ADDRESS_KEY);
  cluster.startDataNodes(conf,1,true,StartupOption.REGULAR,null,null,null,false,true);
  dns=cluster.getDataNodes();
  dn=dns.get(0);
  selfSocketAddr=dn.getXferAddress().toString();
  System.out.println("DN Self Socket Addr == " + selfSocketAddr);
  assertTrue(selfSocketAddr.contains("/127.0.0.1:"));
  for (int i=0; i < dns.size(); i++) {
    DataNodeProperties dnp=cluster.stopDataNode(i);
    assertNotNull("Should have been able to stop simulated datanode",dnp);
  }
  conf.set(DFS_DATANODE_ADDRESS_KEY,"0.0.0.0:0");
  conf.set(DFS_DATANODE_HTTP_ADDRESS_KEY,"0.0.0.0:0");
  conf.set(DFS_DATANODE_IPC_ADDRESS_KEY,"0.0.0.0:0");
  cluster.startDataNodes(conf,1,true,StartupOption.REGULAR,null,null,null,false,true);
  dns=cluster.getDataNodes();
  dn=dns.get(0);
  selfSocketAddr=dn.getXferAddress().toString();
  System.out.println("DN Self Socket Addr == " + selfSocketAddr);
  assertTrue(selfSocketAddr.contains("/0.0.0.0:"));
  cluster.shutdown();
}

Class: org.apache.hadoop.hdfs.TestDFSClientFailover

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Make sure that client failover works when an active NN dies and the standby
 * takes over.
 */
@Test public void testDfsClientFailover() throws IOException, URISyntaxException {
  FileSystem fs=HATestUtil.configureFailoverFs(cluster,conf);
  DFSTestUtil.createFile(fs,TEST_FILE,FILE_LENGTH_TO_VERIFY,(short)1,1L);
  assertEquals(fs.getFileStatus(TEST_FILE).getLen(),FILE_LENGTH_TO_VERIFY);
  cluster.shutdownNameNode(0);
  cluster.transitionToActive(1);
  assertEquals(fs.getFileStatus(TEST_FILE).getLen(),FILE_LENGTH_TO_VERIFY);
  Path withPort=new Path("hdfs://" + HATestUtil.getLogicalHostname(cluster) + ":"+ NameNode.DEFAULT_PORT+ "/"+ TEST_FILE.toUri().getPath());
  FileSystem fs2=withPort.getFileSystem(fs.getConf());
  assertTrue(fs2.exists(withPort));
  fs.close();
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Test to verify legacy proxy providers are correctly wrapped.
 */
@Test public void testWrappedFailoverProxyProvider() throws Exception {
  Configuration config=new HdfsConfiguration(conf);
  String logicalName=HATestUtil.getLogicalHostname(cluster);
  HATestUtil.setFailoverConfigurations(cluster,config,logicalName);
  config.set(DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "." + logicalName,DummyLegacyFailoverProxyProvider.class.getName());
  Path p=new Path("hdfs://" + logicalName + "/");
  SecurityUtil.setTokenServiceUseIp(false);
  assertTrue("Legacy proxy providers should use logical URI.",HAUtil.useLogicalUri(config,p.toUri()));
}

Class: org.apache.hadoop.hdfs.TestDFSClientRetries

APIUtilityVerifier BooleanVerifier 

/** 
 * Test that a DFSClient waits for random time before retry on busy blocks.
 */
@Test public void testDFSClientRetriesOnBusyBlocks() throws IOException {
  System.out.println("Testing DFSClient random waiting on busy blocks.");
  int xcievers=2;
  int fileLen=6 * 1024 * 1024;
  int threads=50;
  int retries=3;
  int timeWin=300;
  long timestamp=Time.now();
  boolean pass=busyTest(xcievers,threads,fileLen,timeWin,retries);
  long timestamp2=Time.now();
  if (pass) {
    LOG.info("Test 1 succeeded! Time spent: " + (timestamp2 - timestamp) / 1000.0 + " sec.");
  }
 else {
    LOG.warn("Test 1 failed, but relax. Time spent: " + (timestamp2 - timestamp) / 1000.0 + " sec.");
  }
  retries=50;
  timestamp=Time.now();
  pass=busyTest(xcievers,threads,fileLen,timeWin,retries);
  timestamp2=Time.now();
  assertTrue("Something wrong! Test 2 got Exception with maxmum retries!",pass);
  LOG.info("Test 2 succeeded! Time spent: " + (timestamp2 - timestamp) / 1000.0 + " sec.");
  retries=3;
  timeWin=1000;
  timestamp=Time.now();
  pass=busyTest(xcievers,threads,fileLen,timeWin,retries);
  timestamp2=Time.now();
  if (pass) {
    LOG.info("Test 3 succeeded! Time spent: " + (timestamp2 - timestamp) / 1000.0 + " sec.");
  }
 else {
    LOG.warn("Test 3 failed, but relax. Time spent: " + (timestamp2 - timestamp) / 1000.0 + " sec.");
  }
  retries=50;
  timeWin=1000;
  timestamp=Time.now();
  pass=busyTest(xcievers,threads,fileLen,timeWin,retries);
  timestamp2=Time.now();
  assertTrue("Something wrong! Test 4 got Exception with maxmum retries!",pass);
  LOG.info("Test 4 succeeded! Time spent: " + (timestamp2 - timestamp) / 1000.0 + " sec.");
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test that getAdditionalBlock() and close() are idempotent. This allows
 * a client to safely retry a call and still produce a correct
 * file. See HDFS-3031.
 */
@Test public void testIdempotentAllocateBlockAndClose() throws Exception {
  final String src="/testIdempotentAllocateBlock";
  Path file=new Path(src);
  conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,4096);
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  try {
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    NamenodeProtocols preSpyNN=cluster.getNameNodeRpc();
    NamenodeProtocols spyNN=spy(preSpyNN);
    DFSClient client=new DFSClient(null,spyNN,conf,null);
    doAnswer(new Answer(){
      @Override public LocatedBlock answer(      InvocationOnMock invocation) throws Throwable {
        LocatedBlock ret=(LocatedBlock)invocation.callRealMethod();
        LocatedBlocks lb=cluster.getNameNodeRpc().getBlockLocations(src,0,Long.MAX_VALUE);
        int blockCount=lb.getLocatedBlocks().size();
        assertEquals(lb.getLastLocatedBlock().getBlock(),ret.getBlock());
        LocatedBlock ret2=(LocatedBlock)invocation.callRealMethod();
        lb=cluster.getNameNodeRpc().getBlockLocations(src,0,Long.MAX_VALUE);
        int blockCount2=lb.getLocatedBlocks().size();
        assertEquals(lb.getLastLocatedBlock().getBlock(),ret2.getBlock());
        assertEquals(blockCount,blockCount2);
        return ret2;
      }
    }
).when(spyNN).addBlock(Mockito.anyString(),Mockito.anyString(),Mockito.any(),Mockito.any(),Mockito.anyLong(),Mockito.any());
    doAnswer(new Answer(){
      @Override public Boolean answer(      InvocationOnMock invocation) throws Throwable {
        LOG.info("Called complete(: " + Joiner.on(",").join(invocation.getArguments()) + ")");
        if (!(Boolean)invocation.callRealMethod()) {
          LOG.info("Complete call returned false, not faking a retry RPC");
          return false;
        }
        try {
          boolean ret=(Boolean)invocation.callRealMethod();
          LOG.info("Complete call returned true, faked second RPC. " + "Returned: " + ret);
          return ret;
        }
 catch (        Throwable t) {
          LOG.error("Idempotent retry threw exception",t);
          throw t;
        }
      }
    }
).when(spyNN).complete(Mockito.anyString(),Mockito.anyString(),Mockito.any(),anyLong());
    OutputStream stm=client.create(file.toString(),true);
    try {
      AppendTestUtil.write(stm,0,10000);
      stm.close();
      stm=null;
    }
  finally {
      IOUtils.cleanup(LOG,stm);
    }
    Mockito.verify(spyNN,Mockito.atLeastOnce()).addBlock(Mockito.anyString(),Mockito.anyString(),Mockito.any(),Mockito.any(),Mockito.anyLong(),Mockito.any());
    Mockito.verify(spyNN,Mockito.atLeastOnce()).complete(Mockito.anyString(),Mockito.anyString(),Mockito.any(),anyLong());
    AppendTestUtil.check(fs,file,10000);
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier IterativeVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that checksum failures are recovered from by the next read on the same
 * DFSInputStream. Corruption information is not persisted from read call to
 * read call, so the client should expect consecutive calls to behave the same
 * way. See HDFS-3067.
 */
@Test public void testRetryOnChecksumFailure() throws Exception {
  HdfsConfiguration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    final short REPL_FACTOR=1;
    final long FILE_LENGTH=512L;
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    Path path=new Path("/corrupted");
    DFSTestUtil.createFile(fs,path,FILE_LENGTH,REPL_FACTOR,12345L);
    DFSTestUtil.waitReplication(fs,path,REPL_FACTOR);
    ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,path);
    int blockFilesCorrupted=cluster.corruptBlockOnDataNodes(block);
    assertEquals("All replicas not corrupted",REPL_FACTOR,blockFilesCorrupted);
    InetSocketAddress nnAddr=new InetSocketAddress("localhost",cluster.getNameNodePort());
    DFSClient client=new DFSClient(nnAddr,conf);
    DFSInputStream dis=client.open(path.toString());
    byte[] arr=new byte[(int)FILE_LENGTH];
    for (int i=0; i < 2; ++i) {
      try {
        dis.read(arr,0,(int)FILE_LENGTH);
        fail("Expected ChecksumException not thrown");
      }
 catch (      Exception ex) {
        GenericTestUtils.assertExceptionContains("Checksum error",ex);
      }
    }
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * This tests that DFSInputStream failures are counted for a given read
 * operation, and not over the lifetime of the stream. It is a regression
 * test for HDFS-127.
 */
@Test public void testFailuresArePerOperation() throws Exception {
  long fileSize=4096;
  Path file=new Path("/testFile");
  conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE,10);
  conf.setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY,2 * 1000);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  try {
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    NamenodeProtocols preSpyNN=cluster.getNameNodeRpc();
    NamenodeProtocols spyNN=spy(preSpyNN);
    DFSClient client=new DFSClient(null,spyNN,conf,null);
    int maxBlockAcquires=client.getMaxBlockAcquireFailures();
    assertTrue(maxBlockAcquires > 0);
    DFSTestUtil.createFile(fs,file,fileSize,(short)1,12345L);
    doAnswer(new FailNTimesAnswer(preSpyNN,maxBlockAcquires + 1)).when(spyNN).getBlockLocations(anyString(),anyLong(),anyLong());
    try {
      IOUtils.copyBytes(client.open(file.toString()),new IOUtils.NullOutputStream(),conf,true);
      fail("Didn't get exception");
    }
 catch (    IOException ioe) {
      DFSClient.LOG.info("Got expected exception",ioe);
    }
    doAnswer(new FailNTimesAnswer(preSpyNN,maxBlockAcquires)).when(spyNN).getBlockLocations(anyString(),anyLong(),anyLong());
    IOUtils.copyBytes(client.open(file.toString()),new IOUtils.NullOutputStream(),conf,true);
    DFSClient.LOG.info("Starting test case for failure reset");
    doAnswer(new FailNTimesAnswer(preSpyNN,maxBlockAcquires)).when(spyNN).getBlockLocations(anyString(),anyLong(),anyLong());
    DFSInputStream is=client.open(file.toString());
    byte buf[]=new byte[10];
    IOUtils.readFully(is,buf,0,buf.length);
    DFSClient.LOG.info("First read successful after some failures.");
    doAnswer(new FailNTimesAnswer(preSpyNN,maxBlockAcquires)).when(spyNN).getBlockLocations(anyString(),anyLong(),anyLong());
    is.openInfo();
    is.seek(0);
    IOUtils.readFully(is,buf,0,buf.length);
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestDFSOutputStream

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * The close() method of DFSOutputStream should never throw the same exception
 * twice. See HDFS-5335 for details.
 */
@Test public void testCloseTwice() throws IOException {
  DistributedFileSystem fs=cluster.getFileSystem();
  FSDataOutputStream os=fs.create(new Path("/test"));
  DFSOutputStream dos=(DFSOutputStream)Whitebox.getInternalState(os,"wrappedStream");
  @SuppressWarnings("unchecked") AtomicReference ex=(AtomicReference)Whitebox.getInternalState(dos,"lastException");
  Assert.assertEquals(null,ex.get());
  dos.close();
  IOException dummy=new IOException("dummy");
  ex.set(dummy);
  try {
    dos.close();
  }
 catch (  IOException e) {
    Assert.assertEquals(e,dummy);
  }
  Assert.assertEquals(null,ex.get());
  dos.close();
}

Class: org.apache.hadoop.hdfs.TestDFSRemove

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testRemove() throws Exception {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  try {
    FileSystem fs=cluster.getFileSystem();
    assertTrue(fs.mkdirs(dir));
    long dfsUsedStart=getTotalDfsUsed(cluster);
{
      final int fileCount=100;
      for (int i=0; i < fileCount; i++) {
        Path a=new Path(dir,"a" + i);
        createFile(fs,a);
      }
      long dfsUsedMax=getTotalDfsUsed(cluster);
      for (int i=0; i < fileCount; i++) {
        Path a=new Path(dir,"a" + i);
        fs.delete(a,false);
      }
      Thread.sleep(3 * DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT * 1000);
      long dfsUsedFinal=getTotalDfsUsed(cluster);
      assertEquals("All blocks should be gone. start=" + dfsUsedStart + " max="+ dfsUsedMax+ " final="+ dfsUsedFinal,dfsUsedStart,dfsUsedFinal);
    }
    fs.delete(dir,true);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestDFSRename

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Check the blocks of dst file are cleaned after rename with overwrite
 */
@Test(timeout=120000) public void testRenameWithOverwrite() throws Exception {
  final short replFactor=2;
  final long blockSize=512;
  Configuration conf=new Configuration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(replFactor).build();
  DistributedFileSystem dfs=cluster.getFileSystem();
  try {
    long fileLen=blockSize * 3;
    String src="/foo/src";
    String dst="/foo/dst";
    Path srcPath=new Path(src);
    Path dstPath=new Path(dst);
    DFSTestUtil.createFile(dfs,srcPath,fileLen,replFactor,1);
    DFSTestUtil.createFile(dfs,dstPath,fileLen,replFactor,1);
    LocatedBlocks lbs=NameNodeAdapter.getBlockLocations(cluster.getNameNode(),dst,0,fileLen);
    BlockManager bm=NameNodeAdapter.getNamesystem(cluster.getNameNode()).getBlockManager();
    assertTrue(bm.getStoredBlock(lbs.getLocatedBlocks().get(0).getBlock().getLocalBlock()) != null);
    dfs.rename(srcPath,dstPath,Rename.OVERWRITE);
    assertTrue(bm.getStoredBlock(lbs.getLocatedBlocks().get(0).getBlock().getLocalBlock()) == null);
  }
  finally {
    if (dfs != null) {
      dfs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestDFSRollback

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier 

/** 
 * This test attempts to rollback the NameNode and DataNode under
 * a number of valid and invalid conditions.
 */
@Test public void testRollback() throws Exception {
  File[] baseDirs;
  UpgradeUtilities.initialize();
  StorageInfo storageInfo=null;
  for (int numDirs=1; numDirs <= 2; numDirs++) {
    conf=new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY,-1);
    conf=UpgradeUtilities.initializeStorageStateConf(numDirs,conf);
    String[] nameNodeDirs=conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
    String[] dataNodeDirs=conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
    log("Normal NameNode rollback",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"previous");
    NameNode.doRollback(conf,false);
    checkResult(NAME_NODE,nameNodeDirs);
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    log("Normal DataNode rollback",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"previous");
    NameNode.doRollback(conf,false);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).manageDataDfsDirs(false).manageNameDfsDirs(false).dnStartupOption(StartupOption.ROLLBACK).build();
    UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs,"current");
    UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs,"previous");
    cluster.startDataNodes(conf,1,false,StartupOption.ROLLBACK,null);
    checkResult(DATA_NODE,dataNodeDirs);
    cluster.shutdown();
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    UpgradeUtilities.createEmptyDirs(dataNodeDirs);
    log("Normal BlockPool rollback",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"previous");
    NameNode.doRollback(conf,false);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).manageDataDfsDirs(false).manageNameDfsDirs(false).dnStartupOption(StartupOption.ROLLBACK).build();
    UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs,"current");
    UpgradeUtilities.createBlockPoolStorageDirs(dataNodeDirs,"current",UpgradeUtilities.getCurrentBlockPoolID(cluster));
    UpgradeUtilities.createBlockPoolStorageDirs(dataNodeDirs,"previous",UpgradeUtilities.getCurrentBlockPoolID(cluster));
    storageInfo=new StorageInfo(HdfsConstants.DATANODE_LAYOUT_VERSION - 1,UpgradeUtilities.getCurrentNamespaceID(cluster),UpgradeUtilities.getCurrentClusterID(cluster),UpgradeUtilities.getCurrentFsscTime(cluster),NodeType.DATA_NODE);
    File[] dataCurrentDirs=new File[dataNodeDirs.length];
    for (int i=0; i < dataNodeDirs.length; i++) {
      dataCurrentDirs[i]=new File((new Path(dataNodeDirs[i] + "/current")).toString());
    }
    UpgradeUtilities.createDataNodeVersionFile(dataCurrentDirs,storageInfo,UpgradeUtilities.getCurrentBlockPoolID(cluster));
    cluster.startDataNodes(conf,1,false,StartupOption.ROLLBACK,null);
    assertTrue(cluster.isDataNodeUp());
    cluster.shutdown();
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    UpgradeUtilities.createEmptyDirs(dataNodeDirs);
    log("NameNode rollback without existing previous dir",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    startNameNodeShouldFail("None of the storage directories contain previous fs state");
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    log("DataNode rollback without existing previous dir",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).manageDataDfsDirs(false).manageNameDfsDirs(false).startupOption(StartupOption.UPGRADE).build();
    UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs,"current");
    cluster.startDataNodes(conf,1,false,StartupOption.ROLLBACK,null);
    cluster.shutdown();
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    UpgradeUtilities.createEmptyDirs(dataNodeDirs);
    log("DataNode rollback with future stored layout version in previous",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"previous");
    NameNode.doRollback(conf,false);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).manageDataDfsDirs(false).manageNameDfsDirs(false).dnStartupOption(StartupOption.ROLLBACK).build();
    UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs,"current");
    baseDirs=UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs,"previous");
    storageInfo=new StorageInfo(Integer.MIN_VALUE,UpgradeUtilities.getCurrentNamespaceID(cluster),UpgradeUtilities.getCurrentClusterID(cluster),UpgradeUtilities.getCurrentFsscTime(cluster),NodeType.DATA_NODE);
    UpgradeUtilities.createDataNodeVersionFile(baseDirs,storageInfo,UpgradeUtilities.getCurrentBlockPoolID(cluster));
    startBlockPoolShouldFail(StartupOption.ROLLBACK,cluster.getNamesystem().getBlockPoolId());
    cluster.shutdown();
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    UpgradeUtilities.createEmptyDirs(dataNodeDirs);
    log("DataNode rollback with newer fsscTime in previous",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"previous");
    NameNode.doRollback(conf,false);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).manageDataDfsDirs(false).manageNameDfsDirs(false).dnStartupOption(StartupOption.ROLLBACK).build();
    UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs,"current");
    baseDirs=UpgradeUtilities.createDataNodeStorageDirs(dataNodeDirs,"previous");
    storageInfo=new StorageInfo(HdfsConstants.DATANODE_LAYOUT_VERSION,UpgradeUtilities.getCurrentNamespaceID(cluster),UpgradeUtilities.getCurrentClusterID(cluster),Long.MAX_VALUE,NodeType.DATA_NODE);
    UpgradeUtilities.createDataNodeVersionFile(baseDirs,storageInfo,UpgradeUtilities.getCurrentBlockPoolID(cluster));
    startBlockPoolShouldFail(StartupOption.ROLLBACK,cluster.getNamesystem().getBlockPoolId());
    cluster.shutdown();
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    UpgradeUtilities.createEmptyDirs(dataNodeDirs);
    log("NameNode rollback with no edits file",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    baseDirs=UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"previous");
    deleteMatchingFiles(baseDirs,"edits.*");
    startNameNodeShouldFail("Gap in transactions");
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    log("NameNode rollback with no image file",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    baseDirs=UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"previous");
    deleteMatchingFiles(baseDirs,"fsimage_.*");
    startNameNodeShouldFail("No valid image files found");
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    log("NameNode rollback with corrupt version file",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    baseDirs=UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"previous");
    for (    File f : baseDirs) {
      UpgradeUtilities.corruptFile(new File(f,"VERSION"),"layoutVersion".getBytes(Charsets.UTF_8),"xxxxxxxxxxxxx".getBytes(Charsets.UTF_8));
    }
    startNameNodeShouldFail("file VERSION has layoutVersion missing");
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    log("NameNode rollback with old layout version in previous",numDirs);
    UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"current");
    baseDirs=UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs,"previous");
    storageInfo=new StorageInfo(1,UpgradeUtilities.getCurrentNamespaceID(null),UpgradeUtilities.getCurrentClusterID(null),UpgradeUtilities.getCurrentFsscTime(null),NodeType.NAME_NODE);
    UpgradeUtilities.createNameNodeVersionFile(conf,baseDirs,storageInfo,UpgradeUtilities.getCurrentBlockPoolID(cluster));
    startNameNodeShouldFail("Cannot rollback to storage version 1 using this version");
    UpgradeUtilities.createEmptyDirs(nameNodeDirs);
  }
}

Class: org.apache.hadoop.hdfs.TestDFSShell

APIUtilityVerifier InternalCallVerifier BooleanVerifier ConditionMatcher HybridVerifier 

@Test(timeout=300000) public void testAppendToFile() throws Exception {
  final int inputFileLength=1024 * 1024;
  File testRoot=new File(TEST_ROOT_DIR,"testAppendtoFileDir");
  testRoot.mkdirs();
  File file1=new File(testRoot,"file1");
  File file2=new File(testRoot,"file2");
  createLocalFileWithRandomData(inputFileLength,file1);
  createLocalFileWithRandomData(inputFileLength,file2);
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  try {
    FileSystem dfs=cluster.getFileSystem();
    assertTrue("Not a HDFS: " + dfs.getUri(),dfs instanceof DistributedFileSystem);
    Path remoteFile=new Path("/remoteFile");
    FsShell shell=new FsShell();
    shell.setConf(conf);
    String[] argv=new String[]{"-appendToFile",file1.toString(),file2.toString(),remoteFile.toString()};
    int res=ToolRunner.run(shell,argv);
    assertThat(res,is(0));
    assertThat(dfs.getFileStatus(remoteFile).getLen(),is((long)inputFileLength * 2));
    res=ToolRunner.run(shell,argv);
    assertThat(res,is(0));
    assertThat(dfs.getFileStatus(remoteFile).getLen(),is((long)inputFileLength * 4));
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testGet() throws IOException {
  DFSTestUtil.setLogLevel2All(FSInputChecker.LOG);
  final String fname="testGet.txt";
  Path root=new Path("/test/get");
  final Path remotef=new Path(root,fname);
  final Configuration conf=new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE,10);
  TestGetRunner runner=new TestGetRunner(){
    private int count=0;
    private final FsShell shell=new FsShell(conf);
    public String run(    int exitcode,    String... options) throws IOException {
      String dst=new File(TEST_ROOT_DIR,fname + ++count).getAbsolutePath();
      String[] args=new String[options.length + 3];
      args[0]="-get";
      args[args.length - 2]=remotef.toString();
      args[args.length - 1]=dst;
      for (int i=0; i < options.length; i++) {
        args[i + 1]=options[i];
      }
      show("args=" + Arrays.asList(args));
      try {
        assertEquals(exitcode,shell.run(args));
      }
 catch (      Exception e) {
        assertTrue(StringUtils.stringifyException(e),false);
      }
      return exitcode == 0 ? DFSTestUtil.readFile(new File(dst)) : null;
    }
  }
;
  File localf=createLocalFile(new File(TEST_ROOT_DIR,fname));
  MiniDFSCluster cluster=null;
  DistributedFileSystem dfs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true).build();
    dfs=cluster.getFileSystem();
    mkdir(dfs,root);
    dfs.copyFromLocalFile(false,false,new Path(localf.getPath()),remotef);
    String localfcontent=DFSTestUtil.readFile(localf);
    assertEquals(localfcontent,runner.run(0));
    assertEquals(localfcontent,runner.run(0,"-ignoreCrc"));
    List files=getBlockFiles(cluster);
    dfs.close();
    cluster.shutdown();
    show("files=" + files);
    corrupt(files);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).format(false).build();
    dfs=cluster.getFileSystem();
    assertEquals(null,runner.run(1));
    String corruptedcontent=runner.run(0,"-ignoreCrc");
    assertEquals(localfcontent.substring(1),corruptedcontent.substring(1));
    assertEquals(localfcontent.charAt(0) + 1,corruptedcontent.charAt(0));
  }
  finally {
    if (null != dfs) {
      try {
        dfs.close();
      }
 catch (      Exception e) {
      }
    }
    if (null != cluster) {
      cluster.shutdown();
    }
    localf.delete();
  }
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test to make sure that user namespace xattrs can be set only if path has
 * access and for sticky directorries, only owner/privileged user can write.
 * Trusted namespace xattrs can be set only with privileged users.
 * As user1: Create a directory (/foo) as user1, chown it to user1 (and
 * user1's group), grant rwx to "other".
 * As user2: Set an xattr (should pass with path access).
 * As user1: Set an xattr (should pass).
 * As user2: Read the xattr (should pass). Remove the xattr (should pass with
 * path access).
 * As user1: Read the xattr (should pass). Remove the xattr (should pass).
 * As user1: Change permissions only to owner
 * As User2: Set an Xattr (Should fail set with no path access) Remove an
 * Xattr (Should fail with no path access)
 * As SuperUser: Set an Xattr with Trusted (Should pass)
 */
@Test(timeout=30000) public void testSetXAttrPermissionAsDifferentOwner() throws Exception {
  final String USER1="user1";
  final String GROUP1="supergroup";
  final UserGroupInformation user1=UserGroupInformation.createUserForTesting(USER1,new String[]{GROUP1});
  final UserGroupInformation user2=UserGroupInformation.createUserForTesting("user2",new String[]{"mygroup2"});
  final UserGroupInformation SUPERUSER=UserGroupInformation.getCurrentUser();
  MiniDFSCluster cluster=null;
  PrintStream bak=null;
  try {
    final Configuration conf=new HdfsConfiguration();
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    final FileSystem fs=cluster.getFileSystem();
    fs.setOwner(new Path("/"),USER1,GROUP1);
    bak=System.err;
    final FsShell fshell=new FsShell(conf);
    final ByteArrayOutputStream out=new ByteArrayOutputStream();
    System.setErr(new PrintStream(out));
    user1.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        final int ret=ToolRunner.run(fshell,new String[]{"-mkdir","/foo"});
        assertEquals("Return should be 0",0,ret);
        out.reset();
        return null;
      }
    }
);
    user1.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        final int ret=ToolRunner.run(fshell,new String[]{"-chmod","707","/foo"});
        assertEquals("Return should be 0",0,ret);
        out.reset();
        return null;
      }
    }
);
    user2.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        final int ret=ToolRunner.run(fshell,new String[]{"-setfattr","-n","user.a1","-v","1234","/foo"});
        assertEquals("Returned should be 0",0,ret);
        out.reset();
        return null;
      }
    }
);
    user1.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        final int ret=ToolRunner.run(fshell,new String[]{"-setfattr","-n","user.a1","-v","1234","/foo"});
        assertEquals("Returned should be 0",0,ret);
        out.reset();
        return null;
      }
    }
);
    user2.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        int ret=ToolRunner.run(fshell,new String[]{"-getfattr","-n","user.a1","/foo"});
        assertEquals("Returned should be 0",0,ret);
        out.reset();
        ret=ToolRunner.run(fshell,new String[]{"-setfattr","-x","user.a1","/foo"});
        assertEquals("Returned should be 0",0,ret);
        out.reset();
        return null;
      }
    }
);
    user1.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        return null;
      }
    }
);
    user1.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        final int ret=ToolRunner.run(fshell,new String[]{"-chmod","700","/foo"});
        assertEquals("Return should be 0",0,ret);
        out.reset();
        return null;
      }
    }
);
    user2.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        int ret=ToolRunner.run(fshell,new String[]{"-setfattr","-n","user.a2","/foo"});
        assertEquals("Returned should be 1",1,ret);
        final String str=out.toString();
        assertTrue("Permission denied printed",str.indexOf("Permission denied") != -1);
        out.reset();
        return null;
      }
    }
);
    user2.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        int ret=ToolRunner.run(fshell,new String[]{"-setfattr","-x","user.a2","/foo"});
        assertEquals("Returned should be 1",1,ret);
        final String str=out.toString();
        assertTrue("Permission denied printed",str.indexOf("Permission denied") != -1);
        out.reset();
        return null;
      }
    }
);
    SUPERUSER.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        int ret=ToolRunner.run(fshell,new String[]{"-setfattr","-n","trusted.a3","/foo"});
        assertEquals("Returned should be 0",0,ret);
        out.reset();
        return null;
      }
    }
);
  }
  finally {
    if (bak != null) {
      System.setErr(bak);
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier ConditionMatcher HybridVerifier 

@Test(timeout=300000) public void testAppendToFileBadArgs() throws Exception {
  final int inputFileLength=1024 * 1024;
  File testRoot=new File(TEST_ROOT_DIR,"testAppendToFileBadArgsDir");
  testRoot.mkdirs();
  File file1=new File(testRoot,"file1");
  createLocalFileWithRandomData(inputFileLength,file1);
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  try {
    FileSystem dfs=cluster.getFileSystem();
    assertTrue("Not a HDFS: " + dfs.getUri(),dfs instanceof DistributedFileSystem);
    FsShell shell=new FsShell();
    shell.setConf(conf);
    String[] argv=new String[]{"-appendToFile",file1.toString()};
    int res=ToolRunner.run(shell,argv);
    assertThat(res,not(0));
    Path remoteFile=new Path("/remoteFile");
    argv=new String[]{"-appendToFile",file1.toString(),"-",remoteFile.toString()};
    res=ToolRunner.run(shell,argv);
    assertThat(res,not(0));
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=30000) public void testPut() throws IOException {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  FileSystem fs=cluster.getFileSystem();
  assertTrue("Not a HDFS: " + fs.getUri(),fs instanceof DistributedFileSystem);
  final DistributedFileSystem dfs=(DistributedFileSystem)fs;
  try {
    new File(TEST_ROOT_DIR,".f1.crc").delete();
    new File(TEST_ROOT_DIR,".f2.crc").delete();
    final File f1=createLocalFile(new File(TEST_ROOT_DIR,"f1"));
    final File f2=createLocalFile(new File(TEST_ROOT_DIR,"f2"));
    final Path root=mkdir(dfs,new Path("/test/put"));
    final Path dst=new Path(root,"dst");
    show("begin");
    final Thread copy2ndFileThread=new Thread(){
      @Override public void run(){
        try {
          show("copy local " + f2 + " to remote "+ dst);
          dfs.copyFromLocalFile(false,false,new Path(f2.getPath()),dst);
        }
 catch (        IOException ioe) {
          show("good " + StringUtils.stringifyException(ioe));
          return;
        }
        assertTrue(false);
      }
    }
;
    SecurityManager sm=System.getSecurityManager();
    System.out.println("SecurityManager = " + sm);
    System.setSecurityManager(new SecurityManager(){
      private boolean firstTime=true;
      @Override public void checkPermission(      Permission perm){
        if (firstTime) {
          Thread t=Thread.currentThread();
          if (!t.toString().contains("DataNode")) {
            String s="" + Arrays.asList(t.getStackTrace());
            if (s.contains("FileUtil.copyContent")) {
              firstTime=false;
              copy2ndFileThread.start();
              try {
                Thread.sleep(5000);
              }
 catch (              InterruptedException e) {
              }
            }
          }
        }
      }
    }
);
    show("copy local " + f1 + " to remote "+ dst);
    dfs.copyFromLocalFile(false,false,new Path(f1.getPath()),dst);
    show("done");
    try {
      copy2ndFileThread.join();
    }
 catch (    InterruptedException e) {
    }
    System.setSecurityManager(sm);
    final Path destmultiple=mkdir(dfs,new Path("/test/putmultiple"));
    Path[] srcs=new Path[2];
    srcs[0]=new Path(f1.getPath());
    srcs[1]=new Path(f2.getPath());
    dfs.copyFromLocalFile(false,false,srcs,destmultiple);
    srcs[0]=new Path(destmultiple,"f1");
    srcs[1]=new Path(destmultiple,"f2");
    assertTrue(dfs.exists(srcs[0]));
    assertTrue(dfs.exists(srcs[1]));
    final Path destmultiple2=mkdir(dfs,new Path("/test/movemultiple"));
    srcs[0]=new Path(f1.getPath());
    srcs[1]=new Path(f2.getPath());
    dfs.moveFromLocalFile(srcs,destmultiple2);
    assertFalse(f1.exists());
    assertFalse(f2.exists());
    srcs[0]=new Path(destmultiple2,"f1");
    srcs[1]=new Path(destmultiple2,"f2");
    assertTrue(dfs.exists(srcs[0]));
    assertTrue(dfs.exists(srcs[1]));
    f1.delete();
    f2.delete();
  }
  finally {
    try {
      dfs.close();
    }
 catch (    Exception e) {
    }
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=30000) public void testDu() throws IOException {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  DistributedFileSystem fs=cluster.getFileSystem();
  PrintStream psBackup=System.out;
  ByteArrayOutputStream out=new ByteArrayOutputStream();
  PrintStream psOut=new PrintStream(out);
  System.setOut(psOut);
  FsShell shell=new FsShell();
  shell.setConf(conf);
  try {
    Path myPath=new Path("/test/dir");
    assertTrue(fs.mkdirs(myPath));
    assertTrue(fs.exists(myPath));
    Path myFile=new Path("/test/dir/file");
    writeFile(fs,myFile);
    assertTrue(fs.exists(myFile));
    Path myFile2=new Path("/test/dir/file2");
    writeFile(fs,myFile2);
    assertTrue(fs.exists(myFile2));
    String[] args=new String[2];
    args[0]="-du";
    args[1]="/test/dir";
    int val=-1;
    try {
      val=shell.run(args);
    }
 catch (    Exception e) {
      System.err.println("Exception raised from DFSShell.run " + e.getLocalizedMessage());
    }
    assertTrue(val == 0);
    String returnString=out.toString();
    out.reset();
    assertTrue(returnString.contains("22"));
    assertTrue(returnString.contains("23"));
  }
  finally {
    System.setOut(psBackup);
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testURIPaths() throws Exception {
  Configuration srcConf=new HdfsConfiguration();
  Configuration dstConf=new HdfsConfiguration();
  MiniDFSCluster srcCluster=null;
  MiniDFSCluster dstCluster=null;
  File bak=new File(PathUtils.getTestDir(getClass()),"dfs_tmp_uri");
  bak.mkdirs();
  try {
    srcCluster=new MiniDFSCluster.Builder(srcConf).numDataNodes(2).build();
    dstConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR,bak.getAbsolutePath());
    dstCluster=new MiniDFSCluster.Builder(dstConf).numDataNodes(2).build();
    FileSystem srcFs=srcCluster.getFileSystem();
    FileSystem dstFs=dstCluster.getFileSystem();
    FsShell shell=new FsShell();
    shell.setConf(srcConf);
    String[] argv=new String[2];
    argv[0]="-ls";
    argv[1]=dstFs.getUri().toString() + "/";
    int ret=ToolRunner.run(shell,argv);
    assertEquals("ls works on remote uri ",0,ret);
    dstFs.mkdirs(new Path("/hadoopdir"));
    argv=new String[2];
    argv[0]="-rmr";
    argv[1]=dstFs.getUri().toString() + "/hadoopdir";
    ret=ToolRunner.run(shell,argv);
    assertEquals("-rmr works on remote uri " + argv[1],0,ret);
    argv[0]="-du";
    argv[1]=dstFs.getUri().toString() + "/";
    ret=ToolRunner.run(shell,argv);
    assertEquals("du works on remote uri ",0,ret);
    File furi=new File(TEST_ROOT_DIR,"furi");
    createLocalFile(furi);
    argv=new String[3];
    argv[0]="-put";
    argv[1]=furi.toURI().toString();
    argv[2]=dstFs.getUri().toString() + "/furi";
    ret=ToolRunner.run(shell,argv);
    assertEquals(" put is working ",0,ret);
    argv[0]="-cp";
    argv[1]=dstFs.getUri().toString() + "/furi";
    argv[2]=srcFs.getUri().toString() + "/furi";
    ret=ToolRunner.run(shell,argv);
    assertEquals(" cp is working ",0,ret);
    assertTrue(srcFs.exists(new Path("/furi")));
    argv=new String[2];
    argv[0]="-cat";
    argv[1]=dstFs.getUri().toString() + "/furi";
    ret=ToolRunner.run(shell,argv);
    assertEquals(" cat is working ",0,ret);
    dstFs.delete(new Path("/furi"),true);
    dstFs.delete(new Path("/hadoopdir"),true);
    String file="/tmp/chownTest";
    Path path=new Path(file);
    Path parent=new Path("/tmp");
    Path root=new Path("/");
    TestDFSShell.writeFile(dstFs,path);
    runCmd(shell,"-chgrp","-R","herbivores",dstFs.getUri().toString() + "/*");
    confirmOwner(null,"herbivores",dstFs,parent,path);
    runCmd(shell,"-chown","-R",":reptiles",dstFs.getUri().toString() + "/");
    confirmOwner(null,"reptiles",dstFs,root,parent,path);
    argv[0]="-cat";
    argv[1]="hdfs:///furi";
    ret=ToolRunner.run(shell,argv);
    assertEquals(" default works for cat",0,ret);
    argv[0]="-ls";
    argv[1]="hdfs:///";
    ret=ToolRunner.run(shell,argv);
    assertEquals("default works for ls ",0,ret);
    argv[0]="-rmr";
    argv[1]="hdfs:///furi";
    ret=ToolRunner.run(shell,argv);
    assertEquals("default works for rm/rmr",0,ret);
  }
  finally {
    if (null != srcCluster) {
      srcCluster.shutdown();
    }
    if (null != dstCluster) {
      dstCluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier ConditionMatcher 

@Test(timeout=30000) public void testSetrep() throws Exception {
  Configuration conf=new Configuration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
  FsShell shell=null;
  FileSystem fs=null;
  final String testdir1="/tmp/TestDFSShell-testSetrep-" + counter.getAndIncrement();
  final String testdir2=testdir1 + "/nestedDir";
  final Path hdfsFile1=new Path(testdir1,"testFileForSetrep");
  final Path hdfsFile2=new Path(testdir2,"testFileForSetrep");
  final Short oldRepFactor=new Short((short)1);
  final Short newRepFactor=new Short((short)3);
  try {
    String[] argv;
    cluster.waitActive();
    fs=cluster.getFileSystem();
    assertThat(fs.mkdirs(new Path(testdir2)),is(true));
    shell=new FsShell(conf);
    fs.create(hdfsFile1,true).close();
    fs.create(hdfsFile2,true).close();
    argv=new String[]{"-setrep",newRepFactor.toString(),hdfsFile1.toString()};
    assertThat(shell.run(argv),is(SUCCESS));
    assertThat(fs.getFileStatus(hdfsFile1).getReplication(),is(newRepFactor));
    assertThat(fs.getFileStatus(hdfsFile2).getReplication(),is(oldRepFactor));
    argv=new String[]{"-setrep",newRepFactor.toString(),testdir1};
    assertThat(shell.run(argv),is(SUCCESS));
    assertThat(fs.getFileStatus(hdfsFile1).getReplication(),is(newRepFactor));
    assertThat(fs.getFileStatus(hdfsFile2).getReplication(),is(newRepFactor));
  }
  finally {
    if (shell != null) {
      shell.close();
    }
    cluster.shutdown();
  }
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testSetXAttrPermission() throws Exception {
  UserGroupInformation user=UserGroupInformation.createUserForTesting("user",new String[]{"mygroup"});
  MiniDFSCluster cluster=null;
  PrintStream bak=null;
  try {
    final Configuration conf=new HdfsConfiguration();
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    Path p=new Path("/foo");
    fs.mkdirs(p);
    bak=System.err;
    final FsShell fshell=new FsShell(conf);
    final ByteArrayOutputStream out=new ByteArrayOutputStream();
    System.setErr(new PrintStream(out));
    fs.setPermission(p,new FsPermission((short)0700));
    user.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        int ret=ToolRunner.run(fshell,new String[]{"-setfattr","-n","user.a1","-v","1234","/foo"});
        assertEquals("Returned should be 1",1,ret);
        String str=out.toString();
        assertTrue("Permission denied printed",str.indexOf("Permission denied") != -1);
        out.reset();
        return null;
      }
    }
);
    int ret=ToolRunner.run(fshell,new String[]{"-setfattr","-n","user.a1","-v","1234","/foo"});
    assertEquals("Returned should be 0",0,ret);
    out.reset();
    fs.setPermission(p,new FsPermission((short)0750));
    user.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        int ret=ToolRunner.run(fshell,new String[]{"-getfattr","-n","user.a1","/foo"});
        assertEquals("Returned should be 1",1,ret);
        String str=out.toString();
        assertTrue("Permission denied printed",str.indexOf("Permission denied") != -1);
        out.reset();
        ret=ToolRunner.run(fshell,new String[]{"-setfattr","-x","user.a1","/foo"});
        assertEquals("Returned should be 1",1,ret);
        str=out.toString();
        assertTrue("Permission denied printed",str.indexOf("Permission denied") != -1);
        out.reset();
        return null;
      }
    }
);
  }
  finally {
    if (bak != null) {
      System.setErr(bak);
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=30000) public void testCount() throws Exception {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  DistributedFileSystem dfs=cluster.getFileSystem();
  FsShell shell=new FsShell();
  shell.setConf(conf);
  try {
    String root=createTree(dfs,"count");
    runCount(root,2,4,shell);
    runCount(root + "2",2,1,shell);
    runCount(root + "2/f1",0,1,shell);
    runCount(root + "2/sub",1,0,shell);
    final FileSystem localfs=FileSystem.getLocal(conf);
    Path localpath=new Path(TEST_ROOT_DIR,"testcount");
    localpath=localpath.makeQualified(localfs.getUri(),localfs.getWorkingDirectory());
    localfs.mkdirs(localpath);
    final String localstr=localpath.toString();
    System.out.println("localstr=" + localstr);
    runCount(localstr,1,0,shell);
    assertEquals(0,runCmd(shell,"-count",root,localstr));
  }
  finally {
    try {
      dfs.close();
    }
 catch (    Exception e) {
    }
    cluster.shutdown();
  }
}

APIUtilityVerifier EqualityVerifier 

@Test(timeout=30000) public void testCopyCommandsWithForceOption() throws Exception {
  Configuration conf=new Configuration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
  FsShell shell=null;
  FileSystem fs=null;
  final File localFile=new File(TEST_ROOT_DIR,"testFileForPut");
  final String localfilepath=new Path(localFile.getAbsolutePath()).toUri().toString();
  final String testdir="/tmp/TestDFSShell-testCopyCommandsWithForceOption-" + counter.getAndIncrement();
  final Path hdfsTestDir=new Path(testdir);
  try {
    fs=cluster.getFileSystem();
    fs.mkdirs(hdfsTestDir);
    localFile.createNewFile();
    writeFile(fs,new Path(testdir,"testFileForPut"));
    shell=new FsShell();
    String[] argv=new String[]{"-put","-f",localfilepath,testdir};
    int res=ToolRunner.run(shell,argv);
    assertEquals("put -f is not working",SUCCESS,res);
    argv=new String[]{"-put",localfilepath,testdir};
    res=ToolRunner.run(shell,argv);
    assertEquals("put command itself is able to overwrite the file",ERROR,res);
    argv=new String[]{"-copyFromLocal","-f",localfilepath,testdir};
    res=ToolRunner.run(shell,argv);
    assertEquals("copyFromLocal -f is not working",SUCCESS,res);
    argv=new String[]{"-copyFromLocal",localfilepath,testdir};
    res=ToolRunner.run(shell,argv);
    assertEquals("copyFromLocal command itself is able to overwrite the file",ERROR,res);
    argv=new String[]{"-cp","-f",localfilepath,testdir};
    res=ToolRunner.run(shell,argv);
    assertEquals("cp -f is not working",SUCCESS,res);
    argv=new String[]{"-cp",localfilepath,testdir};
    res=ToolRunner.run(shell,argv);
    assertEquals("cp command itself is able to overwrite the file",ERROR,res);
  }
  finally {
    if (null != shell)     shell.close();
    if (localFile.exists())     localFile.delete();
    if (null != fs) {
      fs.delete(hdfsTestDir,true);
      fs.close();
    }
    cluster.shutdown();
  }
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testRemoteException() throws Exception {
  UserGroupInformation tmpUGI=UserGroupInformation.createUserForTesting("tmpname",new String[]{"mygroup"});
  MiniDFSCluster dfs=null;
  PrintStream bak=null;
  try {
    final Configuration conf=new HdfsConfiguration();
    dfs=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    FileSystem fs=dfs.getFileSystem();
    Path p=new Path("/foo");
    fs.mkdirs(p);
    fs.setPermission(p,new FsPermission((short)0700));
    bak=System.err;
    tmpUGI.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        FsShell fshell=new FsShell(conf);
        ByteArrayOutputStream out=new ByteArrayOutputStream();
        PrintStream tmp=new PrintStream(out);
        System.setErr(tmp);
        String[] args=new String[2];
        args[0]="-ls";
        args[1]="/foo";
        int ret=ToolRunner.run(fshell,args);
        assertEquals("returned should be 1",1,ret);
        String str=out.toString();
        assertTrue("permission denied printed",str.indexOf("Permission denied") != -1);
        out.reset();
        return null;
      }
    }
);
  }
  finally {
    if (bak != null) {
      System.setErr(bak);
    }
    if (dfs != null) {
      dfs.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * check command error outputs and exit statuses. 
 */
@Test(timeout=30000) public void testErrOutPut() throws Exception {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  PrintStream bak=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    FileSystem srcFs=cluster.getFileSystem();
    Path root=new Path("/nonexistentfile");
    bak=System.err;
    ByteArrayOutputStream out=new ByteArrayOutputStream();
    PrintStream tmp=new PrintStream(out);
    System.setErr(tmp);
    String[] argv=new String[2];
    argv[0]="-cat";
    argv[1]=root.toUri().getPath();
    int ret=ToolRunner.run(new FsShell(),argv);
    assertEquals(" -cat returned 1 ",1,ret);
    String returned=out.toString();
    assertTrue("cat does not print exceptions ",(returned.lastIndexOf("Exception") == -1));
    out.reset();
    argv[0]="-rm";
    argv[1]=root.toString();
    FsShell shell=new FsShell();
    shell.setConf(conf);
    ret=ToolRunner.run(shell,argv);
    assertEquals(" -rm returned 1 ",1,ret);
    returned=out.toString();
    out.reset();
    assertTrue("rm prints reasonable error ",(returned.lastIndexOf("No such file or directory") != -1));
    argv[0]="-rmr";
    argv[1]=root.toString();
    ret=ToolRunner.run(shell,argv);
    assertEquals(" -rmr returned 1",1,ret);
    returned=out.toString();
    assertTrue("rmr prints reasonable error ",(returned.lastIndexOf("No such file or directory") != -1));
    out.reset();
    argv[0]="-du";
    argv[1]="/nonexistentfile";
    ret=ToolRunner.run(shell,argv);
    returned=out.toString();
    assertTrue(" -du prints reasonable error ",(returned.lastIndexOf("No such file or directory") != -1));
    out.reset();
    argv[0]="-dus";
    argv[1]="/nonexistentfile";
    ret=ToolRunner.run(shell,argv);
    returned=out.toString();
    assertTrue(" -dus prints reasonable error",(returned.lastIndexOf("No such file or directory") != -1));
    out.reset();
    argv[0]="-ls";
    argv[1]="/nonexistenfile";
    ret=ToolRunner.run(shell,argv);
    returned=out.toString();
    assertTrue(" -ls does not return Found 0 items",(returned.lastIndexOf("Found 0") == -1));
    out.reset();
    argv[0]="-ls";
    argv[1]="/nonexistentfile";
    ret=ToolRunner.run(shell,argv);
    assertEquals(" -lsr should fail ",1,ret);
    out.reset();
    srcFs.mkdirs(new Path("/testdir"));
    argv[0]="-ls";
    argv[1]="/testdir";
    ret=ToolRunner.run(shell,argv);
    returned=out.toString();
    assertTrue(" -ls does not print out anything ",(returned.lastIndexOf("Found 0") == -1));
    out.reset();
    argv[0]="-ls";
    argv[1]="/user/nonxistant/*";
    ret=ToolRunner.run(shell,argv);
    assertEquals(" -ls on nonexistent glob returns 1",1,ret);
    out.reset();
    argv[0]="-mkdir";
    argv[1]="/testdir";
    ret=ToolRunner.run(shell,argv);
    returned=out.toString();
    assertEquals(" -mkdir returned 1 ",1,ret);
    assertTrue(" -mkdir returned File exists",(returned.lastIndexOf("File exists") != -1));
    Path testFile=new Path("/testfile");
    OutputStream outtmp=srcFs.create(testFile);
    outtmp.write(testFile.toString().getBytes());
    outtmp.close();
    out.reset();
    argv[0]="-mkdir";
    argv[1]="/testfile";
    ret=ToolRunner.run(shell,argv);
    returned=out.toString();
    assertEquals(" -mkdir returned 1",1,ret);
    assertTrue(" -mkdir returned this is a file ",(returned.lastIndexOf("not a directory") != -1));
    out.reset();
    argv=new String[3];
    argv[0]="-mv";
    argv[1]="/testfile";
    argv[2]="file";
    ret=ToolRunner.run(shell,argv);
    assertEquals("mv failed to rename",1,ret);
    out.reset();
    argv=new String[3];
    argv[0]="-mv";
    argv[1]="/testfile";
    argv[2]="/testfiletest";
    ret=ToolRunner.run(shell,argv);
    returned=out.toString();
    assertTrue("no output from rename",(returned.lastIndexOf("Renamed") == -1));
    out.reset();
    argv[0]="-mv";
    argv[1]="/testfile";
    argv[2]="/testfiletmp";
    ret=ToolRunner.run(shell,argv);
    returned=out.toString();
    assertTrue(" unix like output",(returned.lastIndexOf("No such file or") != -1));
    out.reset();
    argv=new String[1];
    argv[0]="-du";
    srcFs.mkdirs(srcFs.getHomeDirectory());
    ret=ToolRunner.run(shell,argv);
    returned=out.toString();
    assertEquals(" no error ",0,ret);
    assertTrue("empty path specified",(returned.lastIndexOf("empty string") == -1));
    out.reset();
    argv=new String[3];
    argv[0]="-test";
    argv[1]="-d";
    argv[2]="/no/such/dir";
    ret=ToolRunner.run(shell,argv);
    returned=out.toString();
    assertEquals(" -test -d wrong result ",1,ret);
    assertTrue(returned.isEmpty());
  }
  finally {
    if (bak != null) {
      System.setErr(bak);
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testCopyToLocal() throws IOException {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  FileSystem fs=cluster.getFileSystem();
  assertTrue("Not a HDFS: " + fs.getUri(),fs instanceof DistributedFileSystem);
  DistributedFileSystem dfs=(DistributedFileSystem)fs;
  FsShell shell=new FsShell();
  shell.setConf(conf);
  try {
    String root=createTree(dfs,"copyToLocal");
{
      try {
        assertEquals(0,runCmd(shell,"-copyToLocal",root + "*",TEST_ROOT_DIR));
      }
 catch (      Exception e) {
        System.err.println("Exception raised from DFSShell.run " + e.getLocalizedMessage());
      }
      File localroot=new File(TEST_ROOT_DIR,"copyToLocal");
      File localroot2=new File(TEST_ROOT_DIR,"copyToLocal2");
      File f1=new File(localroot,"f1");
      assertTrue("Copying failed.",f1.isFile());
      File f2=new File(localroot,"f2");
      assertTrue("Copying failed.",f2.isFile());
      File sub=new File(localroot,"sub");
      assertTrue("Copying failed.",sub.isDirectory());
      File f3=new File(sub,"f3");
      assertTrue("Copying failed.",f3.isFile());
      File f4=new File(sub,"f4");
      assertTrue("Copying failed.",f4.isFile());
      File f5=new File(localroot2,"f1");
      assertTrue("Copying failed.",f5.isFile());
      f1.delete();
      f2.delete();
      f3.delete();
      f4.delete();
      f5.delete();
      sub.delete();
    }
{
      String[] args={"-copyToLocal","nosuchfile",TEST_ROOT_DIR};
      try {
        assertEquals(1,shell.run(args));
      }
 catch (      Exception e) {
        System.err.println("Exception raised from DFSShell.run " + e.getLocalizedMessage());
      }
      File f6=new File(TEST_ROOT_DIR,"nosuchfile");
      assertTrue(!f6.exists());
    }
  }
  finally {
    try {
      dfs.close();
    }
 catch (    Exception e) {
    }
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=120000) public void testCopyCommandsToDirectoryWithPreserveOption() throws Exception {
  Configuration conf=new Configuration();
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,true);
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,true);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
  FsShell shell=null;
  FileSystem fs=null;
  final String testdir="/tmp/TestDFSShell-testCopyCommandsToDirectoryWithPreserveOption-" + counter.getAndIncrement();
  final Path hdfsTestDir=new Path(testdir);
  try {
    fs=cluster.getFileSystem();
    fs.mkdirs(hdfsTestDir);
    Path srcDir=new Path(hdfsTestDir,"srcDir");
    fs.mkdirs(srcDir);
    fs.setAcl(srcDir,Lists.newArrayList(aclEntry(ACCESS,USER,ALL),aclEntry(ACCESS,USER,"foo",ALL),aclEntry(ACCESS,GROUP,READ_EXECUTE),aclEntry(DEFAULT,GROUP,"bar",READ_EXECUTE),aclEntry(ACCESS,OTHER,EXECUTE)));
    fs.setPermission(srcDir,new FsPermission(ALL,READ_EXECUTE,EXECUTE,true));
    Path srcFile=new Path(srcDir,"srcFile");
    fs.create(srcFile).close();
    FileStatus status=fs.getFileStatus(srcDir);
    final long mtime=status.getModificationTime();
    final long atime=status.getAccessTime();
    final String owner=status.getOwner();
    final String group=status.getGroup();
    final FsPermission perm=status.getPermission();
    fs.setXAttr(srcDir,USER_A1,USER_A1_VALUE);
    fs.setXAttr(srcDir,TRUSTED_A1,TRUSTED_A1_VALUE);
    shell=new FsShell(conf);
    Path targetDir1=new Path(hdfsTestDir,"targetDir1");
    String[] argv=new String[]{"-cp","-p",srcDir.toUri().toString(),targetDir1.toUri().toString()};
    int ret=ToolRunner.run(shell,argv);
    assertEquals("cp -p is not working",SUCCESS,ret);
    FileStatus targetStatus=fs.getFileStatus(targetDir1);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    FsPermission targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    Map xattrs=fs.getXAttrs(targetDir1);
    assertTrue(xattrs.isEmpty());
    List acls=fs.getAclStatus(targetDir1).getEntries();
    assertTrue(acls.isEmpty());
    assertFalse(targetPerm.getAclBit());
    Path targetDir2=new Path(hdfsTestDir,"targetDir2");
    argv=new String[]{"-cp","-ptop",srcDir.toUri().toString(),targetDir2.toUri().toString()};
    ret=ToolRunner.run(shell,argv);
    assertEquals("cp -ptop is not working",SUCCESS,ret);
    targetStatus=fs.getFileStatus(targetDir2);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    xattrs=fs.getXAttrs(targetDir2);
    assertTrue(xattrs.isEmpty());
    acls=fs.getAclStatus(targetDir2).getEntries();
    assertTrue(acls.isEmpty());
    assertFalse(targetPerm.getAclBit());
    Path targetDir3=new Path(hdfsTestDir,"targetDir3");
    argv=new String[]{"-cp","-ptopx",srcDir.toUri().toString(),targetDir3.toUri().toString()};
    ret=ToolRunner.run(shell,argv);
    assertEquals("cp -ptopx is not working",SUCCESS,ret);
    targetStatus=fs.getFileStatus(targetDir3);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    xattrs=fs.getXAttrs(targetDir3);
    assertEquals(xattrs.size(),2);
    assertArrayEquals(USER_A1_VALUE,xattrs.get(USER_A1));
    assertArrayEquals(TRUSTED_A1_VALUE,xattrs.get(TRUSTED_A1));
    acls=fs.getAclStatus(targetDir3).getEntries();
    assertTrue(acls.isEmpty());
    assertFalse(targetPerm.getAclBit());
    Path targetDir4=new Path(hdfsTestDir,"targetDir4");
    argv=new String[]{"-cp","-ptopa",srcDir.toUri().toString(),targetDir4.toUri().toString()};
    ret=ToolRunner.run(shell,argv);
    assertEquals("cp -ptopa is not working",SUCCESS,ret);
    targetStatus=fs.getFileStatus(targetDir4);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    xattrs=fs.getXAttrs(targetDir4);
    assertTrue(xattrs.isEmpty());
    acls=fs.getAclStatus(targetDir4).getEntries();
    assertFalse(acls.isEmpty());
    assertTrue(targetPerm.getAclBit());
    assertEquals(fs.getAclStatus(srcDir),fs.getAclStatus(targetDir4));
    Path targetDir5=new Path(hdfsTestDir,"targetDir5");
    argv=new String[]{"-cp","-ptoa",srcDir.toUri().toString(),targetDir5.toUri().toString()};
    ret=ToolRunner.run(shell,argv);
    assertEquals("cp -ptoa is not working",SUCCESS,ret);
    targetStatus=fs.getFileStatus(targetDir5);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    xattrs=fs.getXAttrs(targetDir5);
    assertTrue(xattrs.isEmpty());
    acls=fs.getAclStatus(targetDir5).getEntries();
    assertFalse(acls.isEmpty());
    assertTrue(targetPerm.getAclBit());
    assertEquals(fs.getAclStatus(srcDir),fs.getAclStatus(targetDir5));
  }
  finally {
    if (shell != null) {
      shell.close();
    }
    if (fs != null) {
      fs.delete(hdfsTestDir,true);
      fs.close();
    }
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=120000) public void testCopyCommandsPreserveAclAndStickyBit() throws Exception {
  Configuration conf=new Configuration();
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,true);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
  FsShell shell=null;
  FileSystem fs=null;
  final String testdir="/tmp/TestDFSShell-testCopyCommandsPreserveAclAndStickyBit-" + counter.getAndIncrement();
  final Path hdfsTestDir=new Path(testdir);
  try {
    fs=cluster.getFileSystem();
    fs.mkdirs(hdfsTestDir);
    Path src=new Path(hdfsTestDir,"srcfile");
    fs.create(src).close();
    fs.setAcl(src,Lists.newArrayList(aclEntry(ACCESS,USER,ALL),aclEntry(ACCESS,USER,"foo",ALL),aclEntry(ACCESS,GROUP,READ_EXECUTE),aclEntry(ACCESS,GROUP,"bar",READ_EXECUTE),aclEntry(ACCESS,OTHER,EXECUTE)));
    fs.setPermission(src,new FsPermission(ALL,READ_EXECUTE,EXECUTE,true));
    FileStatus status=fs.getFileStatus(src);
    final long mtime=status.getModificationTime();
    final long atime=status.getAccessTime();
    final String owner=status.getOwner();
    final String group=status.getGroup();
    final FsPermission perm=status.getPermission();
    shell=new FsShell(conf);
    Path target1=new Path(hdfsTestDir,"targetfile1");
    String[] argv=new String[]{"-cp","-p",src.toUri().toString(),target1.toUri().toString()};
    int ret=ToolRunner.run(shell,argv);
    assertEquals("cp is not working",SUCCESS,ret);
    FileStatus targetStatus=fs.getFileStatus(target1);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    FsPermission targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    List acls=fs.getAclStatus(target1).getEntries();
    assertTrue(acls.isEmpty());
    assertFalse(targetPerm.getAclBit());
    Path target2=new Path(hdfsTestDir,"targetfile2");
    argv=new String[]{"-cp","-ptopa",src.toUri().toString(),target2.toUri().toString()};
    ret=ToolRunner.run(shell,argv);
    assertEquals("cp -ptopa is not working",SUCCESS,ret);
    targetStatus=fs.getFileStatus(target2);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    acls=fs.getAclStatus(target2).getEntries();
    assertFalse(acls.isEmpty());
    assertTrue(targetPerm.getAclBit());
    assertEquals(fs.getAclStatus(src),fs.getAclStatus(target2));
  }
  finally {
    if (null != shell) {
      shell.close();
    }
    if (null != fs) {
      fs.delete(hdfsTestDir,true);
      fs.close();
    }
    cluster.shutdown();
  }
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=120000) public void testGetFAttrErrors() throws Exception {
  final UserGroupInformation user=UserGroupInformation.createUserForTesting("user",new String[]{"mygroup"});
  MiniDFSCluster cluster=null;
  PrintStream bakErr=null;
  try {
    final Configuration conf=new HdfsConfiguration();
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    final FileSystem fs=cluster.getFileSystem();
    final Path p=new Path("/foo");
    fs.mkdirs(p);
    bakErr=System.err;
    final FsShell fshell=new FsShell(conf);
    final ByteArrayOutputStream out=new ByteArrayOutputStream();
    System.setErr(new PrintStream(out));
    fs.setPermission(p,new FsPermission((short)0700));
{
      final int ret=ToolRunner.run(fshell,new String[]{"-setfattr","-n","user.a1","-v","1234","/foo"});
      assertEquals("Returned should be 0",0,ret);
      out.reset();
    }
    user.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        int ret=ToolRunner.run(fshell,new String[]{"-getfattr","-n","user.a1","/foo"});
        String str=out.toString();
        assertTrue("xattr value was incorrectly returned",str.indexOf("1234") == -1);
        out.reset();
        return null;
      }
    }
);
{
      final int ret=ToolRunner.run(fshell,new String[]{"-getfattr","-n","user.nonexistent","/foo"});
      String str=out.toString();
      assertTrue("xattr value was incorrectly returned",str.indexOf("getfattr: At least one of the attributes provided was not found") >= 0);
      out.reset();
    }
  }
  finally {
    if (bakErr != null) {
      System.setErr(bakErr);
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=120000) public void testCopyCommandsWithPreserveOption() throws Exception {
  Configuration conf=new Configuration();
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,true);
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,true);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build();
  FsShell shell=null;
  FileSystem fs=null;
  final String testdir="/tmp/TestDFSShell-testCopyCommandsWithPreserveOption-" + counter.getAndIncrement();
  final Path hdfsTestDir=new Path(testdir);
  try {
    fs=cluster.getFileSystem();
    fs.mkdirs(hdfsTestDir);
    Path src=new Path(hdfsTestDir,"srcfile");
    fs.create(src).close();
    fs.setAcl(src,Lists.newArrayList(aclEntry(ACCESS,USER,ALL),aclEntry(ACCESS,USER,"foo",ALL),aclEntry(ACCESS,GROUP,READ_EXECUTE),aclEntry(ACCESS,GROUP,"bar",READ_EXECUTE),aclEntry(ACCESS,OTHER,EXECUTE)));
    FileStatus status=fs.getFileStatus(src);
    final long mtime=status.getModificationTime();
    final long atime=status.getAccessTime();
    final String owner=status.getOwner();
    final String group=status.getGroup();
    final FsPermission perm=status.getPermission();
    fs.setXAttr(src,USER_A1,USER_A1_VALUE);
    fs.setXAttr(src,TRUSTED_A1,TRUSTED_A1_VALUE);
    shell=new FsShell(conf);
    Path target1=new Path(hdfsTestDir,"targetfile1");
    String[] argv=new String[]{"-cp","-p",src.toUri().toString(),target1.toUri().toString()};
    int ret=ToolRunner.run(shell,argv);
    assertEquals("cp -p is not working",SUCCESS,ret);
    FileStatus targetStatus=fs.getFileStatus(target1);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    FsPermission targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    Map xattrs=fs.getXAttrs(target1);
    assertTrue(xattrs.isEmpty());
    List acls=fs.getAclStatus(target1).getEntries();
    assertTrue(acls.isEmpty());
    assertFalse(targetPerm.getAclBit());
    Path target2=new Path(hdfsTestDir,"targetfile2");
    argv=new String[]{"-cp","-ptop",src.toUri().toString(),target2.toUri().toString()};
    ret=ToolRunner.run(shell,argv);
    assertEquals("cp -ptop is not working",SUCCESS,ret);
    targetStatus=fs.getFileStatus(target2);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    xattrs=fs.getXAttrs(target2);
    assertTrue(xattrs.isEmpty());
    acls=fs.getAclStatus(target2).getEntries();
    assertTrue(acls.isEmpty());
    assertFalse(targetPerm.getAclBit());
    Path target3=new Path(hdfsTestDir,"targetfile3");
    argv=new String[]{"-cp","-ptopx",src.toUri().toString(),target3.toUri().toString()};
    ret=ToolRunner.run(shell,argv);
    assertEquals("cp -ptopx is not working",SUCCESS,ret);
    targetStatus=fs.getFileStatus(target3);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    xattrs=fs.getXAttrs(target3);
    assertEquals(xattrs.size(),2);
    assertArrayEquals(USER_A1_VALUE,xattrs.get(USER_A1));
    assertArrayEquals(TRUSTED_A1_VALUE,xattrs.get(TRUSTED_A1));
    acls=fs.getAclStatus(target3).getEntries();
    assertTrue(acls.isEmpty());
    assertFalse(targetPerm.getAclBit());
    Path target4=new Path(hdfsTestDir,"targetfile4");
    argv=new String[]{"-cp","-ptopa",src.toUri().toString(),target4.toUri().toString()};
    ret=ToolRunner.run(shell,argv);
    assertEquals("cp -ptopa is not working",SUCCESS,ret);
    targetStatus=fs.getFileStatus(target4);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    xattrs=fs.getXAttrs(target4);
    assertTrue(xattrs.isEmpty());
    acls=fs.getAclStatus(target4).getEntries();
    assertFalse(acls.isEmpty());
    assertTrue(targetPerm.getAclBit());
    assertEquals(fs.getAclStatus(src),fs.getAclStatus(target4));
    Path target5=new Path(hdfsTestDir,"targetfile5");
    argv=new String[]{"-cp","-ptoa",src.toUri().toString(),target5.toUri().toString()};
    ret=ToolRunner.run(shell,argv);
    assertEquals("cp -ptoa is not working",SUCCESS,ret);
    targetStatus=fs.getFileStatus(target5);
    assertEquals(mtime,targetStatus.getModificationTime());
    assertEquals(atime,targetStatus.getAccessTime());
    assertEquals(owner,targetStatus.getOwner());
    assertEquals(group,targetStatus.getGroup());
    targetPerm=targetStatus.getPermission();
    assertTrue(perm.equals(targetPerm));
    xattrs=fs.getXAttrs(target5);
    assertTrue(xattrs.isEmpty());
    acls=fs.getAclStatus(target5).getEntries();
    assertFalse(acls.isEmpty());
    assertTrue(targetPerm.getAclBit());
    assertEquals(fs.getAclStatus(src),fs.getAclStatus(target5));
  }
  finally {
    if (null != shell) {
      shell.close();
    }
    if (null != fs) {
      fs.delete(hdfsTestDir,true);
      fs.close();
    }
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=30000) public void testLsr() throws Exception {
  final Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  DistributedFileSystem dfs=cluster.getFileSystem();
  try {
    final String root=createTree(dfs,"lsr");
    dfs.mkdirs(new Path(root,"zzz"));
    runLsr(new FsShell(conf),root,0);
    final Path sub=new Path(root,"sub");
    dfs.setPermission(sub,new FsPermission((short)0));
    final UserGroupInformation ugi=UserGroupInformation.getCurrentUser();
    final String tmpusername=ugi.getShortUserName() + "1";
    UserGroupInformation tmpUGI=UserGroupInformation.createUserForTesting(tmpusername,new String[]{tmpusername});
    String results=tmpUGI.doAs(new PrivilegedExceptionAction(){
      @Override public String run() throws Exception {
        return runLsr(new FsShell(conf),root,1);
      }
    }
);
    assertTrue(results.contains("zzz"));
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestDFSStartupVersions

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * This test ensures the appropriate response (successful or failure) from 
 * a Datanode when the system is started with differing version combinations. 
 * 
 * For each 3-tuple in the cross product
 * ({oldLayoutVersion,currentLayoutVersion,futureLayoutVersion},
 * {currentNamespaceId,incorrectNamespaceId},
 * {pastFsscTime,currentFsscTime,futureFsscTime})
 * 1. Startup Namenode with version file containing 
 * (currentLayoutVersion,currentNamespaceId,currentFsscTime)
 * 2. Attempt to startup Datanode with version file containing 
 * this iterations version 3-tuple
 * 
 */
@Test(timeout=300000) public void testVersions() throws Exception {
  UpgradeUtilities.initialize();
  Configuration conf=UpgradeUtilities.initializeStorageStateConf(1,new HdfsConfiguration());
  StorageData[] versions=initializeVersions();
  UpgradeUtilities.createNameNodeStorageDirs(conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY),"current");
  cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).manageDataDfsDirs(false).manageNameDfsDirs(false).startupOption(StartupOption.REGULAR).build();
  StorageData nameNodeVersion=new StorageData(HdfsConstants.NAMENODE_LAYOUT_VERSION,UpgradeUtilities.getCurrentNamespaceID(cluster),UpgradeUtilities.getCurrentClusterID(cluster),UpgradeUtilities.getCurrentFsscTime(cluster),UpgradeUtilities.getCurrentBlockPoolID(cluster));
  log("NameNode version info",NAME_NODE,null,nameNodeVersion);
  String bpid=UpgradeUtilities.getCurrentBlockPoolID(cluster);
  for (int i=0; i < versions.length; i++) {
    File[] storage=UpgradeUtilities.createDataNodeStorageDirs(conf.getStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY),"current");
    log("DataNode version info",DATA_NODE,i,versions[i]);
    UpgradeUtilities.createDataNodeVersionFile(storage,versions[i].storageInfo,bpid,versions[i].blockPoolId);
    try {
      cluster.startDataNodes(conf,1,false,StartupOption.REGULAR,null);
    }
 catch (    Exception ignore) {
    }
    assertTrue(cluster.getNameNode() != null);
    assertEquals(isVersionCompatible(nameNodeVersion,versions[i]),cluster.isDataNodeUp());
    cluster.shutdownDataNodes();
  }
}

Class: org.apache.hadoop.hdfs.TestDFSStorageStateRecovery

APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier BooleanVerifier 

/** 
 * This test iterates over the testCases table for Datanode storage and
 * attempts to startup the DataNode normally.
 */
@Test public void testDNStorageStates() throws Exception {
  String[] baseDirs;
  for (int numDirs=1; numDirs <= 2; numDirs++) {
    conf=new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY,-1);
    conf=UpgradeUtilities.initializeStorageStateConf(numDirs,conf);
    for (int i=0; i < NUM_DN_TEST_CASES; i++) {
      boolean[] testCase=testCases[i];
      boolean shouldRecover=testCase[SHOULD_RECOVER];
      boolean curAfterRecover=testCase[CURRENT_SHOULD_EXIST_AFTER_RECOVER];
      boolean prevAfterRecover=testCase[PREVIOUS_SHOULD_EXIST_AFTER_RECOVER];
      log("DATA_NODE recovery",numDirs,i,testCase);
      createNameNodeStorageState(new boolean[]{true,true,false,false,false});
      cluster=createCluster(conf);
      baseDirs=createDataNodeStorageState(testCase);
      if (!testCase[CURRENT_EXISTS] && !testCase[PREVIOUS_EXISTS] && !testCase[PREVIOUS_TMP_EXISTS]&& !testCase[REMOVED_TMP_EXISTS]) {
        cluster.startDataNodes(conf,1,false,StartupOption.REGULAR,null);
      }
 else {
        if (shouldRecover) {
          cluster.startDataNodes(conf,1,false,StartupOption.REGULAR,null);
          checkResultDataNode(baseDirs,curAfterRecover,prevAfterRecover);
        }
 else {
          cluster.startDataNodes(conf,1,false,StartupOption.REGULAR,null);
          assertFalse(cluster.getDataNodes().get(0).isDatanodeUp());
        }
      }
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier BooleanVerifier 

/** 
 * This test iterates over the testCases table for block pool storage and
 * attempts to startup the DataNode normally.
 */
@Test public void testBlockPoolStorageStates() throws Exception {
  String[] baseDirs;
  String bpid=UpgradeUtilities.getCurrentBlockPoolID(null);
  for (int numDirs=1; numDirs <= 2; numDirs++) {
    conf=new HdfsConfiguration();
    conf.setInt("dfs.datanode.scan.period.hours",-1);
    conf=UpgradeUtilities.initializeStorageStateConf(numDirs,conf);
    for (int i=0; i < NUM_DN_TEST_CASES; i++) {
      boolean[] testCase=testCases[i];
      boolean shouldRecover=testCase[SHOULD_RECOVER];
      boolean curAfterRecover=testCase[CURRENT_SHOULD_EXIST_AFTER_RECOVER];
      boolean prevAfterRecover=testCase[PREVIOUS_SHOULD_EXIST_AFTER_RECOVER];
      log("BLOCK_POOL recovery",numDirs,i,testCase);
      createNameNodeStorageState(new boolean[]{true,true,false,false,false});
      cluster=createCluster(conf);
      baseDirs=createBlockPoolStorageState(bpid,testCase);
      if (!testCase[CURRENT_EXISTS] && !testCase[PREVIOUS_EXISTS] && !testCase[PREVIOUS_TMP_EXISTS]&& !testCase[REMOVED_TMP_EXISTS]) {
        cluster.startDataNodes(conf,1,false,StartupOption.REGULAR,null);
      }
 else {
        if (shouldRecover) {
          cluster.startDataNodes(conf,1,false,StartupOption.REGULAR,null);
          checkResultBlockPool(baseDirs,curAfterRecover,prevAfterRecover);
        }
 else {
          cluster.startDataNodes(conf,1,false,StartupOption.REGULAR,null);
          assertFalse(cluster.getDataNodes().get(0).isBPServiceAlive(bpid));
        }
      }
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestDFSUtil

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testGetPassword() throws Exception {
  File testDir=new File(System.getProperty("test.build.data","target/test-dir"));
  Configuration conf=new Configuration();
  final String ourUrl=JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir+ "/test.jks";
  File file=new File(testDir,"test.jks");
  file.delete();
  conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,ourUrl);
  CredentialProvider provider=CredentialProviderFactory.getProviders(conf).get(0);
  char[] keypass={'k','e','y','p','a','s','s'};
  char[] storepass={'s','t','o','r','e','p','a','s','s'};
  char[] trustpass={'t','r','u','s','t','p','a','s','s'};
  assertEquals(null,provider.getCredentialEntry(DFS_SERVER_HTTPS_KEYPASSWORD_KEY));
  assertEquals(null,provider.getCredentialEntry(DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY));
  assertEquals(null,provider.getCredentialEntry(DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY));
  try {
    provider.createCredentialEntry(DFS_SERVER_HTTPS_KEYPASSWORD_KEY,keypass);
    provider.createCredentialEntry(DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY,storepass);
    provider.createCredentialEntry(DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY,trustpass);
    provider.flush();
  }
 catch (  Exception e) {
    e.printStackTrace();
    throw e;
  }
  assertArrayEquals(keypass,provider.getCredentialEntry(DFS_SERVER_HTTPS_KEYPASSWORD_KEY).getCredential());
  assertArrayEquals(storepass,provider.getCredentialEntry(DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY).getCredential());
  assertArrayEquals(trustpass,provider.getCredentialEntry(DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY).getCredential());
  Assert.assertEquals("keypass",DFSUtil.getPassword(conf,DFS_SERVER_HTTPS_KEYPASSWORD_KEY));
  Assert.assertEquals("storepass",DFSUtil.getPassword(conf,DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY));
  Assert.assertEquals("trustpass",DFSUtil.getPassword(conf,DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY));
  Assert.assertEquals(null,DFSUtil.getPassword(conf,"invalid-alias"));
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testGetNNUris() throws Exception {
  HdfsConfiguration conf=new HdfsConfiguration();
  final String NS1_NN1_ADDR="ns1-nn1.example.com:8020";
  final String NS1_NN2_ADDR="ns1-nn2.example.com:8020";
  final String NS2_NN_ADDR="ns2-nn.example.com:8020";
  final String NN1_ADDR="nn.example.com:8020";
  final String NN1_SRVC_ADDR="nn.example.com:8021";
  final String NN2_ADDR="nn2.example.com:8020";
  conf.set(DFS_NAMESERVICES,"ns1,ns2");
  conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX,"ns1"),"nn1,nn2");
  conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,"ns1","nn1"),NS1_NN1_ADDR);
  conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,"ns1","nn2"),NS1_NN2_ADDR);
  conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,"ns2"),NS2_NN_ADDR);
  conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY,"hdfs://" + NN1_ADDR);
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY,"hdfs://" + NN2_ADDR);
  Collection uris=DFSUtil.getNameServiceUris(conf,DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,DFS_NAMENODE_RPC_ADDRESS_KEY);
  assertEquals(4,uris.size());
  assertTrue(uris.contains(new URI("hdfs://ns1")));
  assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR)));
  assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR)));
  assertTrue(uris.contains(new URI("hdfs://" + NN2_ADDR)));
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY,"viewfs://vfs-name.example.com");
  uris=DFSUtil.getNameServiceUris(conf,DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,DFS_NAMENODE_RPC_ADDRESS_KEY);
  assertEquals(3,uris.size());
  assertTrue(uris.contains(new URI("hdfs://ns1")));
  assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR)));
  assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR)));
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY,"hdfs://ns1");
  uris=DFSUtil.getNameServiceUris(conf,DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,DFS_NAMENODE_RPC_ADDRESS_KEY);
  assertEquals(3,uris.size());
  assertTrue(uris.contains(new URI("hdfs://ns1")));
  assertTrue(uris.contains(new URI("hdfs://" + NS2_NN_ADDR)));
  assertTrue(uris.contains(new URI("hdfs://" + NN1_ADDR)));
  conf=new HdfsConfiguration();
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY,"hdfs://" + NN1_ADDR);
  conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY,NN1_ADDR);
  conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,NN1_SRVC_ADDR);
  uris=DFSUtil.getNameServiceUris(conf,DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,DFS_NAMENODE_RPC_ADDRESS_KEY);
  assertEquals(1,uris.size());
  assertTrue(uris.contains(new URI("hdfs://" + NN1_SRVC_ADDR)));
}

APIUtilityVerifier EqualityVerifier 

@Test public void testGetInfoServer() throws IOException, URISyntaxException {
  HdfsConfiguration conf=new HdfsConfiguration();
  URI httpsport=DFSUtil.getInfoServer(null,conf,"https");
  assertEquals(new URI("https",null,"0.0.0.0",DFS_NAMENODE_HTTPS_PORT_DEFAULT,null,null,null),httpsport);
  URI httpport=DFSUtil.getInfoServer(null,conf,"http");
  assertEquals(new URI("http",null,"0.0.0.0",DFS_NAMENODE_HTTP_PORT_DEFAULT,null,null,null),httpport);
  URI httpAddress=DFSUtil.getInfoServer(new InetSocketAddress("localhost",8020),conf,"http");
  assertEquals(URI.create("http://localhost:" + DFS_NAMENODE_HTTP_PORT_DEFAULT),httpAddress);
}

APIUtilityVerifier EqualityVerifier 

@Test public void testGetHaNnHttpAddresses() throws IOException {
  final String LOGICAL_HOST_NAME="ns1";
  final String NS1_NN1_ADDR="ns1-nn1.example.com:8020";
  final String NS1_NN2_ADDR="ns1-nn2.example.com:8020";
  Configuration conf=createWebHDFSHAConfiguration(LOGICAL_HOST_NAME,NS1_NN1_ADDR,NS1_NN2_ADDR);
  Map> map=DFSUtil.getHaNnWebHdfsAddresses(conf,"webhdfs");
  assertEquals(NS1_NN1_ADDR,map.get("ns1").get("nn1").toString());
  assertEquals(NS1_NN2_ADDR,map.get("ns1").get("nn2").toString());
}

APIUtilityVerifier UtilityVerifier 

/** 
 * Tests for empty configuration, an exception is thrown from{@link DFSUtil#getNNServiceRpcAddresses(Configuration)}{@link DFSUtil#getBackupNodeAddresses(Configuration)}{@link DFSUtil#getSecondaryNameNodeAddresses(Configuration)}
 */
@Test public void testEmptyConf(){
  HdfsConfiguration conf=new HdfsConfiguration(false);
  try {
    Map> map=DFSUtil.getNNServiceRpcAddresses(conf);
    fail("Expected IOException is not thrown, result was: " + DFSUtil.addressMapToString(map));
  }
 catch (  IOException expected) {
  }
  try {
    Map> map=DFSUtil.getBackupNodeAddresses(conf);
    fail("Expected IOException is not thrown, result was: " + DFSUtil.addressMapToString(map));
  }
 catch (  IOException expected) {
  }
  try {
    Map> map=DFSUtil.getSecondaryNameNodeAddresses(conf);
    fail("Expected IOException is not thrown, result was: " + DFSUtil.addressMapToString(map));
  }
 catch (  IOException expected) {
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Tests to ensure default namenode is used as fallback 
 */
@Test public void testDefaultNamenode() throws IOException {
  HdfsConfiguration conf=new HdfsConfiguration();
  final String hdfs_default="hdfs://localhost:9999/";
  conf.set(FS_DEFAULT_NAME_KEY,hdfs_default);
  Map> addrMap=DFSUtil.getNNServiceRpcAddresses(conf);
  assertEquals(1,addrMap.size());
  Map defaultNsMap=addrMap.get(null);
  assertEquals(1,defaultNsMap.size());
  assertEquals(9999,defaultNsMap.get(null).getPort());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test {@link DFSUtil#getNameServiceIds(Configuration)}
 */
@Test public void testGetNameServiceIds(){
  HdfsConfiguration conf=new HdfsConfiguration();
  conf.set(DFS_NAMESERVICES,"nn1,nn2");
  Collection nameserviceIds=DFSUtil.getNameServiceIds(conf);
  Iterator it=nameserviceIds.iterator();
  assertEquals(2,nameserviceIds.size());
  assertEquals("nn1",it.next().toString());
  assertEquals("nn2",it.next().toString());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testHANameNodesWithFederation() throws URISyntaxException {
  HdfsConfiguration conf=new HdfsConfiguration();
  final String NS1_NN1_HOST="ns1-nn1.example.com:8020";
  final String NS1_NN2_HOST="ns1-nn2.example.com:8020";
  final String NS2_NN1_HOST="ns2-nn1.example.com:8020";
  final String NS2_NN2_HOST="ns2-nn2.example.com:8020";
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY,"hdfs://ns1");
  conf.set(DFS_NAMESERVICES,"ns1,ns2");
  conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX,"ns1"),"ns1-nn1,ns1-nn2");
  conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX,"ns2"),"ns2-nn1,ns2-nn2");
  conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,"ns1","ns1-nn1"),NS1_NN1_HOST);
  conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,"ns1","ns1-nn2"),NS1_NN2_HOST);
  conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,"ns2","ns2-nn1"),NS2_NN1_HOST);
  conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,"ns2","ns2-nn2"),NS2_NN2_HOST);
  Map> map=DFSUtil.getHaNnRpcAddresses(conf);
  assertTrue(HAUtil.isHAEnabled(conf,"ns1"));
  assertTrue(HAUtil.isHAEnabled(conf,"ns2"));
  assertFalse(HAUtil.isHAEnabled(conf,"ns3"));
  assertEquals(NS1_NN1_HOST,map.get("ns1").get("ns1-nn1").toString());
  assertEquals(NS1_NN2_HOST,map.get("ns1").get("ns1-nn2").toString());
  assertEquals(NS2_NN1_HOST,map.get("ns2").get("ns2-nn1").toString());
  assertEquals(NS2_NN2_HOST,map.get("ns2").get("ns2-nn2").toString());
  assertEquals(NS1_NN1_HOST,DFSUtil.getNamenodeServiceAddr(conf,"ns1","ns1-nn1"));
  assertEquals(NS1_NN2_HOST,DFSUtil.getNamenodeServiceAddr(conf,"ns1","ns1-nn2"));
  assertEquals(NS2_NN1_HOST,DFSUtil.getNamenodeServiceAddr(conf,"ns2","ns2-nn1"));
  assertEquals(null,DFSUtil.getNamenodeServiceAddr(conf,null,"ns1-nn1"));
  assertEquals(null,DFSUtil.getNamenodeNameServiceId(conf));
  assertEquals(null,DFSUtil.getSecondaryNameServiceId(conf));
  Collection uris=DFSUtil.getNameServiceUris(conf,DFS_NAMENODE_RPC_ADDRESS_KEY);
  assertEquals(2,uris.size());
  assertTrue(uris.contains(new URI("hdfs://ns1")));
  assertTrue(uris.contains(new URI("hdfs://ns2")));
}

APIUtilityVerifier InternalCallVerifier AssumptionSetter EqualityVerifier ConditionMatcher HybridVerifier 

@Test(timeout=15000) public void testLocalhostReverseLookup(){
  Assume.assumeTrue(!Shell.WINDOWS);
  HdfsConfiguration conf=new HdfsConfiguration();
  conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY,"hdfs://127.0.0.1:8020");
  Collection uris=DFSUtil.getNameServiceUris(conf);
  assertEquals(1,uris.size());
  for (  URI uri : uris) {
    assertThat(uri.getHost(),not("127.0.0.1"));
  }
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test conversion of LocatedBlock to BlockLocation
 */
@Test public void testLocatedBlocks2Locations(){
  DatanodeInfo d=DFSTestUtil.getLocalDatanodeInfo();
  DatanodeInfo[] ds=new DatanodeInfo[1];
  ds[0]=d;
  ExtendedBlock b1=new ExtendedBlock("bpid",1,1,1);
  LocatedBlock l1=new LocatedBlock(b1,ds,0,false);
  ExtendedBlock b2=new ExtendedBlock("bpid",2,1,1);
  LocatedBlock l2=new LocatedBlock(b2,ds,0,true);
  List ls=Arrays.asList(l1,l2);
  LocatedBlocks lbs=new LocatedBlocks(10,false,ls,l2,true,null);
  BlockLocation[] bs=DFSUtil.locatedBlocks2Locations(lbs);
  assertTrue("expected 2 blocks but got " + bs.length,bs.length == 2);
  int corruptCount=0;
  for (  BlockLocation b : bs) {
    if (b.isCorrupt()) {
      corruptCount++;
    }
  }
  assertTrue("expected 1 corrupt files but got " + corruptCount,corruptCount == 1);
  bs=DFSUtil.locatedBlocks2Locations(new LocatedBlocks());
  assertEquals(0,bs.length);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test for {@link DFSUtil#getNNServiceRpcAddresses(Configuration)}{@link DFSUtil#getNameServiceIdFromAddress(Configuration,InetSocketAddress,String)(Configuration)}
 */
@Test public void testMultipleNamenodes() throws IOException {
  HdfsConfiguration conf=new HdfsConfiguration();
  conf.set(DFS_NAMESERVICES,"nn1,nn2");
  final String NN1_ADDRESS="localhost:9000";
  final String NN2_ADDRESS="localhost:9001";
  final String NN3_ADDRESS="localhost:9002";
  conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,"nn1"),NN1_ADDRESS);
  conf.set(DFSUtil.addKeySuffixes(DFS_NAMENODE_RPC_ADDRESS_KEY,"nn2"),NN2_ADDRESS);
  Map> nnMap=DFSUtil.getNNServiceRpcAddresses(conf);
  assertEquals(2,nnMap.size());
  Map nn1Map=nnMap.get("nn1");
  assertEquals(1,nn1Map.size());
  InetSocketAddress addr=nn1Map.get(null);
  assertEquals("localhost",addr.getHostName());
  assertEquals(9000,addr.getPort());
  Map nn2Map=nnMap.get("nn2");
  assertEquals(1,nn2Map.size());
  addr=nn2Map.get(null);
  assertEquals("localhost",addr.getHostName());
  assertEquals(9001,addr.getPort());
  checkNameServiceId(conf,NN1_ADDRESS,"nn1");
  checkNameServiceId(conf,NN2_ADDRESS,"nn2");
  checkNameServiceId(conf,NN3_ADDRESS,null);
  assertFalse(HAUtil.isHAEnabled(conf,"nn1"));
  assertFalse(HAUtil.isHAEnabled(conf,"nn2"));
}

Class: org.apache.hadoop.hdfs.TestDataTransferKeepalive

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that the client respects its keepalive timeout.
 */
@Test(timeout=30000) public void testClientResponsesKeepAliveTimeout() throws Exception {
  Configuration clientConf=new Configuration(conf);
  final long CLIENT_EXPIRY_MS=10L;
  clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY,CLIENT_EXPIRY_MS);
  clientConf.set(DFS_CLIENT_CONTEXT,"testClientResponsesKeepAliveTimeout");
  DistributedFileSystem fs=(DistributedFileSystem)FileSystem.get(cluster.getURI(),clientConf);
  PeerCache peerCache=ClientContext.getFromConf(clientConf).getPeerCache();
  DFSTestUtil.createFile(fs,TEST_FILE,1L,(short)1,0L);
  assertEquals(0,peerCache.size());
  assertXceiverCount(0);
  DFSTestUtil.readFile(fs,TEST_FILE);
  assertEquals(1,peerCache.size());
  assertXceiverCount(1);
  Thread.sleep(CLIENT_EXPIRY_MS + 1);
  Peer peer=peerCache.get(dn.getDatanodeId(),false);
  assertTrue(peer == null);
  assertEquals(0,peerCache.size());
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Regression test for HDFS-3357. Check that the datanode is respecting
 * its configured keepalive timeout.
 */
@Test(timeout=30000) public void testDatanodeRespectsKeepAliveTimeout() throws Exception {
  Configuration clientConf=new Configuration(conf);
  final long CLIENT_EXPIRY_MS=60000L;
  clientConf.setLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY,CLIENT_EXPIRY_MS);
  clientConf.set(DFS_CLIENT_CONTEXT,"testDatanodeRespectsKeepAliveTimeout");
  DistributedFileSystem fs=(DistributedFileSystem)FileSystem.get(cluster.getURI(),clientConf);
  PeerCache peerCache=ClientContext.getFromConf(clientConf).getPeerCache();
  DFSTestUtil.createFile(fs,TEST_FILE,1L,(short)1,0L);
  assertEquals(0,peerCache.size());
  assertXceiverCount(0);
  DFSTestUtil.readFile(fs,TEST_FILE);
  assertEquals(1,peerCache.size());
  assertXceiverCount(1);
  Thread.sleep(DFS_DATANODE_SOCKET_REUSE_KEEPALIVE_DEFAULT + 1);
  assertXceiverCount(0);
  assertEquals(1,peerCache.size());
  Peer peer=peerCache.get(dn.getDatanodeId(),false);
  assertNotNull(peer);
  assertEquals(-1,peer.getInputStream().read());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=30000) public void testManyClosedSocketsInCache() throws Exception {
  Configuration clientConf=new Configuration(conf);
  clientConf.set(DFS_CLIENT_CONTEXT,"testManyClosedSocketsInCache");
  DistributedFileSystem fs=(DistributedFileSystem)FileSystem.get(cluster.getURI(),clientConf);
  PeerCache peerCache=ClientContext.getFromConf(clientConf).getPeerCache();
  DFSTestUtil.createFile(fs,TEST_FILE,1L,(short)1,0L);
  InputStream[] stms=new InputStream[5];
  try {
    for (int i=0; i < stms.length; i++) {
      stms[i]=fs.open(TEST_FILE);
    }
    for (    InputStream stm : stms) {
      IOUtils.copyBytes(stm,new NullOutputStream(),1024);
    }
  }
  finally {
    IOUtils.cleanup(null,stms);
  }
  assertEquals(5,peerCache.size());
  Thread.sleep(1500);
  assertXceiverCount(0);
  assertEquals(5,peerCache.size());
  DFSTestUtil.readFile(fs,TEST_FILE);
}

Class: org.apache.hadoop.hdfs.TestDatanodeBlockScanner

APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testDuplicateScans() throws Exception {
  long startTime=Time.monotonicNow();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).build();
  FileSystem fs=null;
  try {
    fs=cluster.getFileSystem();
    DataNode dataNode=cluster.getDataNodes().get(0);
    int infoPort=dataNode.getInfoPort();
    long scanTimeBefore=0, scanTimeAfter=0;
    for (int i=1; i < 10; i++) {
      Path fileName=new Path("/test" + i);
      DFSTestUtil.createFile(fs,fileName,1024,(short)1,1000L);
      waitForVerification(infoPort,fs,fileName,i,startTime,TIMEOUT);
      if (i > 1) {
        scanTimeAfter=DataNodeTestUtils.getLatestScanTime(dataNode,DFSTestUtil.getFirstBlock(fs,new Path("/test" + (i - 1))));
        assertFalse("scan time shoud not be 0",scanTimeAfter == 0);
        assertEquals("There should not be duplicate scan",scanTimeBefore,scanTimeAfter);
      }
      scanTimeBefore=DataNodeTestUtils.getLatestScanTime(dataNode,DFSTestUtil.getFirstBlock(fs,new Path("/test" + i)));
    }
    cluster.restartDataNode(0);
    Thread.sleep(10000);
    dataNode=cluster.getDataNodes().get(0);
    scanTimeAfter=DataNodeTestUtils.getLatestScanTime(dataNode,DFSTestUtil.getFirstBlock(fs,new Path("/test" + (9))));
    assertEquals("There should not be duplicate scan",scanTimeBefore,scanTimeAfter);
  }
  finally {
    IOUtils.closeStream(fs);
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testBlockCorruptionPolicy() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,1000L);
  Random random=new Random();
  FileSystem fs=null;
  int rand=random.nextInt(3);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
  cluster.waitActive();
  fs=cluster.getFileSystem();
  Path file1=new Path("/tmp/testBlockVerification/file1");
  DFSTestUtil.createFile(fs,file1,1024,(short)3,0);
  ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,file1);
  DFSTestUtil.waitReplication(fs,file1,(short)3);
  assertFalse(DFSTestUtil.allBlockReplicasCorrupt(cluster,file1,0));
  assertTrue(MiniDFSCluster.corruptReplica(rand,block));
  cluster.restartDataNode(rand);
  DFSTestUtil.waitReplication(fs,file1,(short)2);
  assertFalse(DFSTestUtil.allBlockReplicasCorrupt(cluster,file1,0));
  assertTrue(MiniDFSCluster.corruptReplica(0,block));
  assertTrue(MiniDFSCluster.corruptReplica(1,block));
  assertTrue(MiniDFSCluster.corruptReplica(2,block));
  for (  DataNode dn : cluster.getDataNodes()) {
    DataNodeTestUtils.runBlockScannerForBlock(dn,block);
  }
  DFSTestUtil.waitReplication(fs,file1,(short)3);
  assertTrue(DFSTestUtil.allBlockReplicasCorrupt(cluster,file1,0));
  cluster.shutdown();
}

Class: org.apache.hadoop.hdfs.TestDatanodeConfig

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier HybridVerifier 

/** 
 * Test that a data-node does not start if configuration specifies
 * incorrect URI scheme in data directory.
 * Test that a data-node starts if data directory is specified as
 * URI = "file:///path" or as a non URI path.
 */
@Test public void testDataDirectories() throws IOException {
  File dataDir=new File(BASE_DIR,"data").getCanonicalFile();
  Configuration conf=cluster.getConfiguration(0);
  String dnDir=makeURI("shv",null,fileAsURI(dataDir).getPath());
  conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,dnDir);
  DataNode dn=null;
  try {
    dn=DataNode.createDataNode(new String[]{},conf);
    fail();
  }
 catch (  Exception e) {
  }
 finally {
    if (dn != null) {
      dn.shutdown();
    }
  }
  assertNull("Data-node startup should have failed.",dn);
  String dnDir1=fileAsURI(dataDir).toString() + "1";
  String dnDir2=makeURI("file","localhost",fileAsURI(dataDir).getPath() + "2");
  String dnDir3=dataDir.getAbsolutePath() + "3";
  conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,dnDir1 + "," + dnDir2+ ","+ dnDir3);
  try {
    cluster.startDataNodes(conf,1,false,StartupOption.REGULAR,null);
    assertTrue("Data-node should startup.",cluster.isDataNodeUp());
  }
  finally {
    if (cluster != null) {
      cluster.shutdownDataNodes();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestDecommission

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests decommission with replicas on the target datanode cannot be migrated
 * to other datanodes and satisfy the replication factor. Make sure the
 * datanode won't get stuck in decommissioning state.
 */
@Test(timeout=360000) public void testDecommission2() throws IOException {
  LOG.info("Starting test testDecommission");
  int numNamenodes=1;
  int numDatanodes=4;
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY,3);
  startCluster(numNamenodes,numDatanodes,conf);
  ArrayList> namenodeDecomList=new ArrayList>(numNamenodes);
  namenodeDecomList.add(0,new ArrayList(numDatanodes));
  Path file1=new Path("testDecommission2.dat");
  int replicas=4;
  ArrayList decommissionedNodes=namenodeDecomList.get(0);
  FileSystem fileSys=cluster.getFileSystem(0);
  FSNamesystem ns=cluster.getNamesystem(0);
  writeFile(fileSys,file1,replicas);
  int deadDecomissioned=ns.getNumDecomDeadDataNodes();
  int liveDecomissioned=ns.getNumDecomLiveDataNodes();
  DatanodeInfo decomNode=decommissionNode(0,null,decommissionedNodes,AdminStates.DECOMMISSIONED);
  decommissionedNodes.add(decomNode);
  assertEquals(deadDecomissioned,ns.getNumDecomDeadDataNodes());
  assertEquals(liveDecomissioned + 1,ns.getNumDecomLiveDataNodes());
  DFSClient client=getDfsClient(cluster.getNameNode(0),conf);
  assertEquals("All datanodes must be alive",numDatanodes,client.datanodeReport(DatanodeReportType.LIVE).length);
  assertNull(checkFile(fileSys,file1,replicas,decomNode.getXferAddr(),numDatanodes));
  cleanupFile(fileSys,file1);
  cluster.shutdown();
  startCluster(1,4,conf);
  cluster.shutdown();
}

APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test using a "registration name" in a host include file.
 * Registration names are DataNode names specified in the configuration by
 * dfs.datanode.hostname.  The DataNode will send this name to the NameNode
 * as part of its registration.  Registration names are helpful when you
 * want to override the normal first result of DNS resolution on the
 * NameNode.  For example, a given datanode IP may map to two hostnames,
 * and you may want to choose which hostname is used internally in the
 * cluster.
 * It is not recommended to use a registration name which is not also a
 * valid DNS hostname for the DataNode.  See HDFS-5237 for background.
 */
@Test(timeout=360000) public void testIncludeByRegistrationName() throws IOException, InterruptedException {
  Configuration hdfsConf=new Configuration(conf);
  final String registrationName="127.0.0.100";
  final String nonExistentDn="127.0.0.10";
  hdfsConf.set(DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY,registrationName);
  cluster=new MiniDFSCluster.Builder(hdfsConf).numDataNodes(1).checkDataNodeHostConfig(true).setupHostsFile(true).build();
  cluster.waitActive();
  ArrayList nodes=new ArrayList();
  nodes.add(nonExistentDn);
  writeConfigFile(hostsFile,nodes);
  refreshNodes(cluster.getNamesystem(0),hdfsConf);
  DFSClient client=getDfsClient(cluster.getNameNode(0),hdfsConf);
  while (true) {
    DatanodeInfo info[]=client.datanodeReport(DatanodeReportType.DEAD);
    if (info.length == 1) {
      break;
    }
    LOG.info("Waiting for datanode to be marked dead");
    Thread.sleep(HEARTBEAT_INTERVAL * 1000);
  }
  int dnPort=cluster.getDataNodes().get(0).getXferPort();
  nodes=new ArrayList();
  nodes.add(registrationName + ":" + dnPort);
  writeConfigFile(hostsFile,nodes);
  refreshNodes(cluster.getNamesystem(0),hdfsConf);
  cluster.restartDataNode(0);
  while (true) {
    DatanodeInfo info[]=client.datanodeReport(DatanodeReportType.LIVE);
    if (info.length == 1) {
      Assert.assertFalse(info[0].isDecommissioned());
      Assert.assertFalse(info[0].isDecommissionInProgress());
      assertEquals(registrationName,info[0].getHostName());
      break;
    }
    LOG.info("Waiting for datanode to come back");
    Thread.sleep(HEARTBEAT_INTERVAL * 1000);
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests restart of namenode while datanode hosts are added to exclude file
 */
@Test(timeout=360000) public void testDecommissionWithNamenodeRestart() throws IOException, InterruptedException {
  LOG.info("Starting test testDecommissionWithNamenodeRestart");
  int numNamenodes=1;
  int numDatanodes=1;
  int replicas=1;
  startCluster(numNamenodes,numDatanodes,conf);
  Path file1=new Path("testDecommission.dat");
  FileSystem fileSys=cluster.getFileSystem();
  writeFile(fileSys,file1,replicas);
  DFSClient client=getDfsClient(cluster.getNameNode(),conf);
  DatanodeInfo[] info=client.datanodeReport(DatanodeReportType.LIVE);
  DatanodeID excludedDatanodeID=info[0];
  String excludedDatanodeName=info[0].getXferAddr();
  writeConfigFile(excludeFile,new ArrayList(Arrays.asList(excludedDatanodeName)));
  cluster.startDataNodes(conf,1,true,null,null,null,null);
  numDatanodes+=1;
  assertEquals("Number of datanodes should be 2 ",2,cluster.getDataNodes().size());
  cluster.restartNameNode();
  DatanodeInfo datanodeInfo=NameNodeAdapter.getDatanode(cluster.getNamesystem(),excludedDatanodeID);
  waitNodeState(datanodeInfo,AdminStates.DECOMMISSIONED);
  assertEquals("All datanodes must be alive",numDatanodes,client.datanodeReport(DatanodeReportType.LIVE).length);
  int tries=0;
  while (tries++ < 20) {
    try {
      Thread.sleep(1000);
      if (checkFile(fileSys,file1,replicas,datanodeInfo.getXferAddr(),numDatanodes) == null) {
        break;
      }
    }
 catch (    InterruptedException ie) {
    }
  }
  assertTrue("Checked if block was replicated after decommission, tried " + tries + " times.",tries < 20);
  cleanupFile(fileSys,file1);
  cluster.shutdown();
  startCluster(numNamenodes,numDatanodes,conf);
  cluster.shutdown();
}

Class: org.apache.hadoop.hdfs.TestDistributedFileSystem

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests the normal path of batching up BlockLocation[]s to be passed to a
 * single{@link DistributedFileSystem#getFileBlockStorageLocations(java.util.List)}call
 */
@Test(timeout=60000) public void testGetFileBlockStorageLocationsBatching() throws Exception {
  final Configuration conf=getTestConfiguration();
  ((Log4JLogger)ProtobufRpcEngine.LOG).getLogger().setLevel(Level.TRACE);
  ((Log4JLogger)BlockStorageLocationUtil.LOG).getLogger().setLevel(Level.TRACE);
  ((Log4JLogger)DFSClient.LOG).getLogger().setLevel(Level.TRACE);
  conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,true);
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  try {
    final DistributedFileSystem fs=cluster.getFileSystem();
    final Path tmpFile1=new Path("/tmpfile1.dat");
    final Path tmpFile2=new Path("/tmpfile2.dat");
    DFSTestUtil.createFile(fs,tmpFile1,1024,(short)2,0xDEADDEADl);
    DFSTestUtil.createFile(fs,tmpFile2,1024,(short)2,0xDEADDEADl);
    GenericTestUtils.waitFor(new Supplier(){
      @Override public Boolean get(){
        try {
          List list=Lists.newArrayList();
          list.addAll(Arrays.asList(fs.getFileBlockLocations(tmpFile1,0,1024)));
          list.addAll(Arrays.asList(fs.getFileBlockLocations(tmpFile2,0,1024)));
          int totalRepl=0;
          for (          BlockLocation loc : list) {
            totalRepl+=loc.getHosts().length;
          }
          if (totalRepl == 4) {
            return true;
          }
        }
 catch (        IOException e) {
        }
        return false;
      }
    }
,500,30000);
    BlockLocation[] blockLocs1=fs.getFileBlockLocations(tmpFile1,0,1024);
    BlockLocation[] blockLocs2=fs.getFileBlockLocations(tmpFile2,0,1024);
    BlockLocation[] blockLocs=(BlockLocation[])ArrayUtils.addAll(blockLocs1,blockLocs2);
    BlockStorageLocation[] locs=fs.getFileBlockStorageLocations(Arrays.asList(blockLocs));
    int counter=0;
    for (    BlockStorageLocation l : locs) {
      for (int i=0; i < l.getVolumeIds().length; i++) {
        VolumeId id=l.getVolumeIds()[i];
        String name=l.getNames()[i];
        if (id != null) {
          System.out.println("Datanode " + name + " has block "+ counter+ " on volume id "+ id.toString());
        }
      }
      counter++;
    }
    assertEquals("Expected two HdfsBlockLocations for two 1-block files",2,locs.length);
    for (    BlockStorageLocation l : locs) {
      assertEquals("Expected two replicas for each block",2,l.getVolumeIds().length);
      for (int i=0; i < l.getVolumeIds().length; i++) {
        VolumeId id=l.getVolumeIds()[i];
        String name=l.getNames()[i];
        assertTrue("Expected block to be valid on datanode " + name,id != null);
      }
    }
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestEncryptedTransfer

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testLongLivedClient() throws IOException, InterruptedException {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new Configuration();
    cluster=new MiniDFSCluster.Builder(conf).build();
    FileSystem fs=getFileSystem(conf);
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    FileChecksum checksum=fs.getFileChecksum(TEST_PATH);
    fs.close();
    cluster.shutdown();
    setEncryptionConfigKeys(conf);
    cluster=new MiniDFSCluster.Builder(conf).manageDataDfsDirs(false).manageNameDfsDirs(false).format(false).startupOption(StartupOption.REGULAR).build();
    BlockTokenSecretManager btsm=cluster.getNamesystem().getBlockManager().getBlockTokenSecretManager();
    btsm.setKeyUpdateIntervalForTesting(2 * 1000);
    btsm.setTokenLifetime(2 * 1000);
    btsm.clearAllKeysForTesting();
    fs=getFileSystem(conf);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    assertEquals(checksum,fs.getFileChecksum(TEST_PATH));
    LOG.info("Sleeping so that encryption keys expire...");
    Thread.sleep(15 * 1000);
    LOG.info("Done sleeping.");
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    assertEquals(checksum,fs.getFileChecksum(TEST_PATH));
    fs.close();
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testLongLivedReadClientAfterRestart() throws IOException {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new Configuration();
    cluster=new MiniDFSCluster.Builder(conf).build();
    FileSystem fs=getFileSystem(conf);
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    FileChecksum checksum=fs.getFileChecksum(TEST_PATH);
    fs.close();
    cluster.shutdown();
    setEncryptionConfigKeys(conf);
    cluster=new MiniDFSCluster.Builder(conf).manageDataDfsDirs(false).manageNameDfsDirs(false).format(false).startupOption(StartupOption.REGULAR).build();
    fs=getFileSystem(conf);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    assertEquals(checksum,fs.getFileChecksum(TEST_PATH));
    cluster.restartNameNode();
    assertTrue(cluster.restartDataNode(0));
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    assertEquals(checksum,fs.getFileChecksum(TEST_PATH));
    fs.close();
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testEncryptedRead() throws IOException {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new Configuration();
    cluster=new MiniDFSCluster.Builder(conf).build();
    FileSystem fs=getFileSystem(conf);
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    FileChecksum checksum=fs.getFileChecksum(TEST_PATH);
    fs.close();
    cluster.shutdown();
    setEncryptionConfigKeys(conf);
    cluster=new MiniDFSCluster.Builder(conf).manageDataDfsDirs(false).manageNameDfsDirs(false).format(false).startupOption(StartupOption.REGULAR).build();
    fs=getFileSystem(conf);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    assertEquals(checksum,fs.getFileChecksum(TEST_PATH));
    fs.close();
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testEncryptedAppendRequiringBlockTransfer() throws IOException {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new Configuration();
    setEncryptionConfigKeys(conf);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    FileSystem fs=getFileSystem(conf);
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    FSDataInputStream in=fs.open(TEST_PATH);
    List locatedBlocks=DFSTestUtil.getAllBlocks(in);
    in.close();
    assertEquals(1,locatedBlocks.size());
    assertEquals(3,locatedBlocks.get(0).getLocations().length);
    DataNode dn=cluster.getDataNode(locatedBlocks.get(0).getLocations()[0].getIpcPort());
    dn.shutdown();
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT + PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    fs.close();
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testEncryptedReadAfterNameNodeRestart() throws IOException {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new Configuration();
    cluster=new MiniDFSCluster.Builder(conf).build();
    FileSystem fs=getFileSystem(conf);
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    FileChecksum checksum=fs.getFileChecksum(TEST_PATH);
    fs.close();
    cluster.shutdown();
    setEncryptionConfigKeys(conf);
    cluster=new MiniDFSCluster.Builder(conf).manageDataDfsDirs(false).manageNameDfsDirs(false).format(false).startupOption(StartupOption.REGULAR).build();
    fs=getFileSystem(conf);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    assertEquals(checksum,fs.getFileChecksum(TEST_PATH));
    fs.close();
    cluster.restartNameNode();
    fs=getFileSystem(conf);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    assertEquals(checksum,fs.getFileChecksum(TEST_PATH));
    fs.close();
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testLongLivedWriteClientAfterRestart() throws IOException {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new Configuration();
    setEncryptionConfigKeys(conf);
    cluster=new MiniDFSCluster.Builder(conf).build();
    FileSystem fs=getFileSystem(conf);
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    cluster.restartNameNode();
    assertTrue(cluster.restartDataNodes());
    cluster.waitActive();
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT + PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    fs.close();
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testEncryptedReadWithRC4() throws IOException {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new Configuration();
    cluster=new MiniDFSCluster.Builder(conf).build();
    FileSystem fs=getFileSystem(conf);
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    FileChecksum checksum=fs.getFileChecksum(TEST_PATH);
    fs.close();
    cluster.shutdown();
    setEncryptionConfigKeys(conf);
    conf.set(DFSConfigKeys.DFS_DATA_ENCRYPTION_ALGORITHM_KEY,"rc4");
    cluster=new MiniDFSCluster.Builder(conf).manageDataDfsDirs(false).manageNameDfsDirs(false).format(false).startupOption(StartupOption.REGULAR).build();
    fs=getFileSystem(conf);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    assertEquals(checksum,fs.getFileChecksum(TEST_PATH));
    fs.close();
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier BranchVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

@Test public void testClientThatDoesNotSupportEncryption() throws IOException {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new Configuration();
    conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE,10);
    cluster=new MiniDFSCluster.Builder(conf).build();
    FileSystem fs=getFileSystem(conf);
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    fs.close();
    cluster.shutdown();
    setEncryptionConfigKeys(conf);
    cluster=new MiniDFSCluster.Builder(conf).manageDataDfsDirs(false).manageNameDfsDirs(false).format(false).startupOption(StartupOption.REGULAR).build();
    fs=getFileSystem(conf);
    DFSClient client=DFSClientAdapter.getDFSClient((DistributedFileSystem)fs);
    DFSClient spyClient=Mockito.spy(client);
    Mockito.doReturn(false).when(spyClient).shouldEncryptData();
    DFSClientAdapter.setDFSClient((DistributedFileSystem)fs,spyClient);
    LogCapturer logs=GenericTestUtils.LogCapturer.captureLogs(LogFactory.getLog(DataNode.class));
    try {
      assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
      if (resolverClazz != null && !resolverClazz.endsWith("TestTrustedChannelResolver")) {
        fail("Should not have been able to read without encryption enabled.");
      }
    }
 catch (    IOException ioe) {
      GenericTestUtils.assertExceptionContains("Could not obtain block:",ioe);
    }
 finally {
      logs.stopCapturing();
    }
    fs.close();
    if (resolverClazz != null && !resolverClazz.endsWith("TestTrustedChannelResolver")) {
      GenericTestUtils.assertMatches(logs.getOutput(),"Failed to read expected encryption handshake from client at");
    }
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void testEncryptedAppend() throws IOException {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new Configuration();
    setEncryptionConfigKeys(conf);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    FileSystem fs=getFileSystem(conf);
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    writeTestDataToFile(fs);
    assertEquals(PLAIN_TEXT + PLAIN_TEXT,DFSTestUtil.readFile(fs,TEST_PATH));
    fs.close();
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestEncryptionZones

APIUtilityVerifier UtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test getEncryptionZoneForPath as a non super user.
 */
@Test(timeout=60000) public void testGetEZAsNonSuperUser() throws Exception {
  final UserGroupInformation user=UserGroupInformation.createUserForTesting("user",new String[]{"mygroup"});
  final Path testRoot=new Path(fsHelper.getTestRootDir());
  final Path superPath=new Path(testRoot,"superuseronly");
  final Path superPathFile=new Path(superPath,"file1");
  final Path allPath=new Path(testRoot,"accessall");
  final Path allPathFile=new Path(allPath,"file1");
  final Path nonEZDir=new Path(testRoot,"nonEZDir");
  final Path nonEZFile=new Path(nonEZDir,"file1");
  final int len=8192;
  fsWrapper.mkdir(testRoot,new FsPermission((short)0777),true);
  fsWrapper.mkdir(superPath,new FsPermission((short)0700),false);
  fsWrapper.mkdir(allPath,new FsPermission((short)0777),false);
  fsWrapper.mkdir(nonEZDir,new FsPermission((short)0777),false);
  dfsAdmin.createEncryptionZone(superPath,TEST_KEY);
  dfsAdmin.createEncryptionZone(allPath,TEST_KEY);
  dfsAdmin.allowSnapshot(new Path("/"));
  final Path newSnap=fs.createSnapshot(new Path("/"));
  DFSTestUtil.createFile(fs,superPathFile,len,(short)1,0xFEED);
  DFSTestUtil.createFile(fs,allPathFile,len,(short)1,0xFEED);
  DFSTestUtil.createFile(fs,nonEZFile,len,(short)1,0xFEED);
  user.doAs(new PrivilegedExceptionAction(){
    @Override public Object run() throws Exception {
      final HdfsAdmin userAdmin=new HdfsAdmin(FileSystem.getDefaultUri(conf),conf);
      try {
        userAdmin.getEncryptionZoneForPath(null);
        fail("should have thrown NPE");
      }
 catch (      NullPointerException e) {
      }
      assertEquals("expected ez path",allPath.toString(),userAdmin.getEncryptionZoneForPath(allPath).getPath().toString());
      assertEquals("expected ez path",allPath.toString(),userAdmin.getEncryptionZoneForPath(allPathFile).getPath().toString());
      try {
        userAdmin.getEncryptionZoneForPath(superPathFile);
        fail("expected AccessControlException");
      }
 catch (      AccessControlException e) {
        assertExceptionContains("Permission denied:",e);
      }
      assertNull("expected null for non-ez path",userAdmin.getEncryptionZoneForPath(nonEZDir));
      assertNull("expected null for non-ez path",userAdmin.getEncryptionZoneForPath(nonEZFile));
      String snapshottedAllPath=newSnap.toString() + allPath.toString();
      assertEquals("expected ez path",allPath.toString(),userAdmin.getEncryptionZoneForPath(new Path(snapshottedAllPath)).getPath().toString());
      fs.delete(allPathFile,false);
      assertEquals("expected ez path",allPath.toString(),userAdmin.getEncryptionZoneForPath(new Path(snapshottedAllPath)).getPath().toString());
      fs.delete(allPath,true);
      assertEquals("expected ez path",allPath.toString(),userAdmin.getEncryptionZoneForPath(new Path(snapshottedAllPath)).getPath().toString());
      assertNull("expected null for deleted file path",userAdmin.getEncryptionZoneForPath(allPathFile));
      assertNull("expected null for deleted directory path",userAdmin.getEncryptionZoneForPath(allPath));
      return null;
    }
  }
);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=120000) public void testReadWrite() throws Exception {
  final HdfsAdmin dfsAdmin=new HdfsAdmin(FileSystem.getDefaultUri(conf),conf);
  final Path baseFile=new Path("/base");
  final int len=8192;
  DFSTestUtil.createFile(fs,baseFile,len,(short)1,0xFEED);
  final Path zone=new Path("/zone");
  fs.mkdirs(zone);
  dfsAdmin.createEncryptionZone(zone,TEST_KEY);
  final Path encFile1=new Path(zone,"myfile");
  DFSTestUtil.createFile(fs,encFile1,len,(short)1,0xFEED);
  verifyFilesEqual(fs,baseFile,encFile1,len);
  assertNumZones(1);
  String keyName=dfsAdmin.listEncryptionZones().next().getKeyName();
  cluster.getNamesystem().getProvider().rollNewVersion(keyName);
  verifyFilesEqual(fs,baseFile,encFile1,len);
  final Path encFile2=new Path(zone,"myfile2");
  DFSTestUtil.createFile(fs,encFile2,len,(short)1,0xFEED);
  FileEncryptionInfo feInfo1=getFileEncryptionInfo(encFile1);
  FileEncryptionInfo feInfo2=getFileEncryptionInfo(encFile2);
  assertFalse("EDEKs should be different",Arrays.equals(feInfo1.getEncryptedDataEncryptionKey(),feInfo2.getEncryptedDataEncryptionKey()));
  assertNotEquals("Key was rolled, versions should be different",feInfo1.getEzKeyVersionName(),feInfo2.getEzKeyVersionName());
  verifyFilesEqual(fs,encFile1,encFile2,len);
}

APIUtilityVerifier IterativeVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

@Test(timeout=60000) public void testCipherSuiteNegotiation() throws Exception {
  final HdfsAdmin dfsAdmin=new HdfsAdmin(FileSystem.getDefaultUri(conf),conf);
  final Path zone=new Path("/zone");
  fs.mkdirs(zone);
  dfsAdmin.createEncryptionZone(zone,TEST_KEY);
  DFSTestUtil.createFile(fs,new Path(zone,"success1"),0,(short)1,0xFEED);
  fs.getClient().cipherSuites=Lists.newArrayListWithCapacity(0);
  try {
    DFSTestUtil.createFile(fs,new Path(zone,"fail"),0,(short)1,0xFEED);
    fail("Created a file without specifying a CipherSuite!");
  }
 catch (  UnknownCipherSuiteException e) {
    assertExceptionContains("No cipher suites",e);
  }
  fs.getClient().cipherSuites=Lists.newArrayListWithCapacity(3);
  fs.getClient().cipherSuites.add(CipherSuite.UNKNOWN);
  fs.getClient().cipherSuites.add(CipherSuite.UNKNOWN);
  fs.getClient().cipherSuites.add(CipherSuite.UNKNOWN);
  try {
    DFSTestUtil.createFile(fs,new Path(zone,"fail"),0,(short)1,0xFEED);
    fail("Created a file without specifying a CipherSuite!");
  }
 catch (  UnknownCipherSuiteException e) {
    assertExceptionContains("No cipher suites",e);
  }
  fs.getClient().cipherSuites=Lists.newArrayListWithCapacity(3);
  fs.getClient().cipherSuites.add(CipherSuite.AES_CTR_NOPADDING);
  fs.getClient().cipherSuites.add(CipherSuite.UNKNOWN);
  fs.getClient().cipherSuites.add(CipherSuite.UNKNOWN);
  DFSTestUtil.createFile(fs,new Path(zone,"success2"),0,(short)1,0xFEED);
  fs.getClient().cipherSuites=Lists.newArrayListWithCapacity(3);
  fs.getClient().cipherSuites.add(CipherSuite.UNKNOWN);
  fs.getClient().cipherSuites.add(CipherSuite.UNKNOWN);
  fs.getClient().cipherSuites.add(CipherSuite.AES_CTR_NOPADDING);
  DFSTestUtil.createFile(fs,new Path(zone,"success3"),4096,(short)1,0xFEED);
  cluster.getNamesystem().getProvider().flush();
  KeyProvider provider=KeyProviderFactory.getProviders(conf).get(0);
  List keys=provider.getKeys();
  assertEquals("Expected NN to have created one key per zone",1,keys.size());
  List allVersions=Lists.newArrayList();
  for (  String key : keys) {
    List versions=provider.getKeyVersions(key);
    assertEquals("Should only have one key version per key",1,versions.size());
    allVersions.addAll(versions);
  }
  for (int i=2; i <= 3; i++) {
    FileEncryptionInfo feInfo=getFileEncryptionInfo(new Path(zone.toString() + "/success" + i));
    assertEquals(feInfo.getCipherSuite(),CipherSuite.AES_CTR_NOPADDING);
  }
}

Class: org.apache.hadoop.hdfs.TestFileAppend

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Test that copy on write for blocks works correctly
 * @throws IOException an exception might be thrown
 */
@Test public void testCopyOnWrite() throws IOException {
  Configuration conf=new HdfsConfiguration();
  if (simulatedStorage) {
    SimulatedFSDataset.setFactory(conf);
  }
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  FileSystem fs=cluster.getFileSystem();
  InetSocketAddress addr=new InetSocketAddress("localhost",cluster.getNameNodePort());
  DFSClient client=new DFSClient(addr,conf);
  try {
    Path file1=new Path("/filestatus.dat");
    FSDataOutputStream stm=AppendTestUtil.createFile(fs,file1,1);
    writeFile(stm);
    stm.close();
    DataNode[] dn=cluster.listDataNodes();
    assertTrue("There should be only one datanode but found " + dn.length,dn.length == 1);
    LocatedBlocks locations=client.getNamenode().getBlockLocations(file1.toString(),0,Long.MAX_VALUE);
    List blocks=locations.getLocatedBlocks();
    for (int i=0; i < blocks.size(); i=i + 2) {
      ExtendedBlock b=blocks.get(i).getBlock();
      final File f=DataNodeTestUtils.getFile(dn[0],b.getBlockPoolId(),b.getLocalBlock().getBlockId());
      File link=new File(f.toString() + ".link");
      System.out.println("Creating hardlink for File " + f + " to "+ link);
      HardLink.createHardLink(f,link);
    }
    for (int i=0; i < blocks.size(); i++) {
      ExtendedBlock b=blocks.get(i).getBlock();
      System.out.println("testCopyOnWrite detaching block " + b);
      assertTrue("Detaching block " + b + " should have returned true",DataNodeTestUtils.unlinkBlock(dn[0],b,1));
    }
    for (int i=0; i < blocks.size(); i++) {
      ExtendedBlock b=blocks.get(i).getBlock();
      System.out.println("testCopyOnWrite detaching block " + b);
      assertTrue("Detaching block " + b + " should have returned false",!DataNodeTestUtils.unlinkBlock(dn[0],b,1));
    }
  }
  finally {
    client.close();
    fs.close();
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestFileAppend3

APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * TC11: Racing rename
 * @throws IOException an exception might be thrown
 */
@Test public void testTC11() throws Exception {
  final Path p=new Path("/TC11/foo");
  System.out.println("p=" + p);
  final int len1=(int)BLOCK_SIZE;
{
    FSDataOutputStream out=fs.create(p,false,buffersize,REPLICATION,BLOCK_SIZE);
    AppendTestUtil.write(out,0,len1);
    out.close();
  }
  FSDataOutputStream out=fs.append(p);
  final int len2=(int)BLOCK_SIZE / 2;
  AppendTestUtil.write(out,len1,len2);
  out.hflush();
  final Path pnew=new Path(p + ".new");
  assertTrue(fs.rename(p,pnew));
  out.close();
  final long len=fs.getFileStatus(pnew).getLen();
  final LocatedBlocks locatedblocks=fs.dfs.getNamenode().getBlockLocations(pnew.toString(),0L,len);
  final int numblock=locatedblocks.locatedBlockCount();
  for (int i=0; i < numblock; i++) {
    final LocatedBlock lb=locatedblocks.get(i);
    final ExtendedBlock blk=lb.getBlock();
    final long size=lb.getBlockSize();
    if (i < numblock - 1) {
      assertEquals(BLOCK_SIZE,size);
    }
    for (    DatanodeInfo datanodeinfo : lb.getLocations()) {
      final DataNode dn=cluster.getDataNode(datanodeinfo.getIpcPort());
      final Block metainfo=DataNodeTestUtils.getFSDataset(dn).getStoredBlock(blk.getBlockPoolId(),blk.getBlockId());
      assertEquals(size,metainfo.getNumBytes());
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier 

/** 
 * TC7: Corrupted replicas are present.
 * @throws IOException an exception might be thrown
 */
@Test public void testTC7() throws Exception {
  final short repl=2;
  final Path p=new Path("/TC7/foo");
  System.out.println("p=" + p);
  final int len1=(int)(BLOCK_SIZE / 2);
{
    FSDataOutputStream out=fs.create(p,false,buffersize,repl,BLOCK_SIZE);
    AppendTestUtil.write(out,0,len1);
    out.close();
  }
  DFSTestUtil.waitReplication(fs,p,repl);
  final LocatedBlocks locatedblocks=fs.dfs.getNamenode().getBlockLocations(p.toString(),0L,len1);
  assertEquals(1,locatedblocks.locatedBlockCount());
  final LocatedBlock lb=locatedblocks.get(0);
  final ExtendedBlock blk=lb.getBlock();
  assertEquals(len1,lb.getBlockSize());
  DatanodeInfo[] datanodeinfos=lb.getLocations();
  assertEquals(repl,datanodeinfos.length);
  final DataNode dn=cluster.getDataNode(datanodeinfos[0].getIpcPort());
  final File f=DataNodeTestUtils.getBlockFile(dn,blk.getBlockPoolId(),blk.getLocalBlock());
  final RandomAccessFile raf=new RandomAccessFile(f,"rw");
  AppendTestUtil.LOG.info("dn=" + dn + ", blk="+ blk+ " (length="+ blk.getNumBytes()+ ")");
  assertEquals(len1,raf.length());
  raf.setLength(0);
  raf.close();
  final int len2=(int)BLOCK_SIZE;
{
    FSDataOutputStream out=fs.append(p);
    AppendTestUtil.write(out,len1,len2);
    out.close();
  }
  AppendTestUtil.check(fs,p,len1 + len2);
}

Class: org.apache.hadoop.hdfs.TestFileAppend4

APIUtilityVerifier BooleanVerifier NullVerifier HybridVerifier 

/** 
 * Test case that stops a writer after finalizing a block but
 * before calling completeFile, and then tries to recover
 * the lease from another thread.
 */
@Test(timeout=60000) public void testRecoverFinalizedBlock() throws Throwable {
  cluster=new MiniDFSCluster.Builder(conf).numDataNodes(5).build();
  try {
    cluster.waitActive();
    NamenodeProtocols preSpyNN=cluster.getNameNodeRpc();
    NamenodeProtocols spyNN=spy(preSpyNN);
    GenericTestUtils.DelayAnswer delayer=new GenericTestUtils.DelayAnswer(LOG);
    doAnswer(delayer).when(spyNN).complete(anyString(),anyString(),(ExtendedBlock)anyObject(),anyLong());
    DFSClient client=new DFSClient(null,spyNN,conf,null);
    file1=new Path("/testRecoverFinalized");
    final OutputStream stm=client.create("/testRecoverFinalized",true);
    AppendTestUtil.write(stm,0,4096);
    final AtomicReference err=new AtomicReference();
    Thread t=new Thread(){
      @Override public void run(){
        try {
          stm.close();
        }
 catch (        Throwable t) {
          err.set(t);
        }
      }
    }
;
    t.start();
    LOG.info("Waiting for close to get to latch...");
    delayer.waitForCall();
    LOG.info("Killing lease checker");
    client.getLeaseRenewer().interruptAndJoin();
    FileSystem fs1=cluster.getFileSystem();
    FileSystem fs2=AppendTestUtil.createHdfsWithDifferentUsername(fs1.getConf());
    LOG.info("Recovering file");
    recoverFile(fs2);
    LOG.info("Telling close to proceed.");
    delayer.proceed();
    LOG.info("Waiting for close to finish.");
    t.join();
    LOG.info("Close finished.");
    Throwable thrownByClose=err.get();
    assertNotNull(thrownByClose);
    assertTrue(thrownByClose instanceof IOException);
    if (!thrownByClose.getMessage().contains("No lease on /testRecoverFinalized"))     throw thrownByClose;
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier BooleanVerifier NullVerifier HybridVerifier 

/** 
 * Test case that stops a writer after finalizing a block but
 * before calling completeFile, recovers a file from another writer,
 * starts writing from that writer, and then has the old lease holder
 * call completeFile
 */
@Test(timeout=60000) public void testCompleteOtherLeaseHoldersFile() throws Throwable {
  cluster=new MiniDFSCluster.Builder(conf).numDataNodes(5).build();
  try {
    cluster.waitActive();
    NamenodeProtocols preSpyNN=cluster.getNameNodeRpc();
    NamenodeProtocols spyNN=spy(preSpyNN);
    GenericTestUtils.DelayAnswer delayer=new GenericTestUtils.DelayAnswer(LOG);
    doAnswer(delayer).when(spyNN).complete(anyString(),anyString(),(ExtendedBlock)anyObject(),anyLong());
    DFSClient client=new DFSClient(null,spyNN,conf,null);
    file1=new Path("/testCompleteOtherLease");
    final OutputStream stm=client.create("/testCompleteOtherLease",true);
    AppendTestUtil.write(stm,0,4096);
    final AtomicReference err=new AtomicReference();
    Thread t=new Thread(){
      @Override public void run(){
        try {
          stm.close();
        }
 catch (        Throwable t) {
          err.set(t);
        }
      }
    }
;
    t.start();
    LOG.info("Waiting for close to get to latch...");
    delayer.waitForCall();
    LOG.info("Killing lease checker");
    client.getLeaseRenewer().interruptAndJoin();
    FileSystem fs1=cluster.getFileSystem();
    FileSystem fs2=AppendTestUtil.createHdfsWithDifferentUsername(fs1.getConf());
    LOG.info("Recovering file");
    recoverFile(fs2);
    LOG.info("Opening file for append from new fs");
    FSDataOutputStream appenderStream=fs2.append(file1);
    LOG.info("Writing some data from new appender");
    AppendTestUtil.write(appenderStream,0,4096);
    LOG.info("Telling old close to proceed.");
    delayer.proceed();
    LOG.info("Waiting for close to finish.");
    t.join();
    LOG.info("Close finished.");
    Throwable thrownByClose=err.get();
    assertNotNull(thrownByClose);
    assertTrue(thrownByClose instanceof IOException);
    if (!thrownByClose.getMessage().contains("Lease mismatch"))     throw thrownByClose;
    appenderStream.close();
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestFileAppendRestart

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Regression test for HDFS-2991. Creates and appends to files
 * where blocks start/end on block boundaries.
 */
@Test public void testAppendRestart() throws Exception {
  final Configuration conf=new HdfsConfiguration();
  conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,0);
  MiniDFSCluster cluster=null;
  FSDataOutputStream stream=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    FileSystem fs=cluster.getFileSystem();
    File editLog=new File(FSImageTestUtil.getNameNodeCurrentDirs(cluster,0).get(0),NNStorage.getInProgressEditsFileName(1));
    EnumMap> counts;
    Path p1=new Path("/block-boundaries");
    writeAndAppend(fs,p1,BLOCK_SIZE,BLOCK_SIZE);
    counts=FSImageTestUtil.countEditLogOpTypes(editLog);
    assertEquals(2,(int)counts.get(FSEditLogOpCodes.OP_ADD).held);
    assertEquals(2,(int)counts.get(FSEditLogOpCodes.OP_ADD_BLOCK).held);
    assertEquals(2,(int)counts.get(FSEditLogOpCodes.OP_CLOSE).held);
    Path p2=new Path("/not-block-boundaries");
    writeAndAppend(fs,p2,BLOCK_SIZE / 2,BLOCK_SIZE);
    counts=FSImageTestUtil.countEditLogOpTypes(editLog);
    assertEquals(2 + 2,(int)counts.get(FSEditLogOpCodes.OP_ADD).held);
    assertEquals(1,(int)counts.get(FSEditLogOpCodes.OP_UPDATE_BLOCKS).held);
    assertEquals(2 + 2,(int)counts.get(FSEditLogOpCodes.OP_ADD_BLOCK).held);
    assertEquals(2 + 2,(int)counts.get(FSEditLogOpCodes.OP_CLOSE).held);
    cluster.restartNameNode();
    AppendTestUtil.check(fs,p1,2 * BLOCK_SIZE);
    AppendTestUtil.check(fs,p2,3 * BLOCK_SIZE / 2);
  }
  finally {
    IOUtils.closeStream(stream);
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestFileCorruption

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test the case that a replica is reported corrupt while it is not
 * in blocksMap. Make sure that ArrayIndexOutOfBounds does not thrown.
 * See Hadoop-4351.
 */
@Test public void testArrayOutOfBoundsException() throws Exception {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new HdfsConfiguration();
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    final Path FILE_PATH=new Path("/tmp.txt");
    final long FILE_LEN=1L;
    DFSTestUtil.createFile(fs,FILE_PATH,FILE_LEN,(short)2,1L);
    final String bpid=cluster.getNamesystem().getBlockPoolId();
    File storageDir=cluster.getInstanceStorageDir(0,0);
    File dataDir=MiniDFSCluster.getFinalizedDir(storageDir,bpid);
    assertTrue("Data directory does not exist",dataDir.exists());
    ExtendedBlock blk=getBlock(bpid,dataDir);
    if (blk == null) {
      storageDir=cluster.getInstanceStorageDir(0,1);
      dataDir=MiniDFSCluster.getFinalizedDir(storageDir,bpid);
      blk=getBlock(bpid,dataDir);
    }
    assertFalse("Data directory does not contain any blocks or there was an " + "IO error",blk == null);
    cluster.startDataNodes(conf,1,true,null,null);
    ArrayList datanodes=cluster.getDataNodes();
    assertEquals(datanodes.size(),3);
    DataNode dataNode=datanodes.get(2);
    DatanodeRegistration dnR=DataNodeTestUtils.getDNRegistrationForBP(dataNode,blk.getBlockPoolId());
    FSNamesystem ns=cluster.getNamesystem();
    ns.writeLock();
    try {
      cluster.getNamesystem().getBlockManager().findAndMarkBlockAsCorrupt(blk,new DatanodeInfo(dnR),"TEST","STORAGE_ID");
    }
  finally {
      ns.writeUnlock();
    }
    fs.open(FILE_PATH);
    fs.delete(FILE_PATH,false);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier 

/** 
 * check if DFS can handle corrupted blocks properly 
 */
@Test public void testFileCorruption() throws Exception {
  MiniDFSCluster cluster=null;
  DFSTestUtil util=new DFSTestUtil.Builder().setName("TestFileCorruption").setNumFiles(20).build();
  try {
    Configuration conf=new HdfsConfiguration();
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    FileSystem fs=cluster.getFileSystem();
    util.createFiles(fs,"/srcdat");
    File storageDir=cluster.getInstanceStorageDir(2,0);
    String bpid=cluster.getNamesystem().getBlockPoolId();
    File data_dir=MiniDFSCluster.getFinalizedDir(storageDir,bpid);
    assertTrue("data directory does not exist",data_dir.exists());
    File[] blocks=data_dir.listFiles();
    assertTrue("Blocks do not exist in data-dir",(blocks != null) && (blocks.length > 0));
    for (int idx=0; idx < blocks.length; idx++) {
      if (!blocks[idx].getName().startsWith("blk_")) {
        continue;
      }
      System.out.println("Deliberately removing file " + blocks[idx].getName());
      assertTrue("Cannot remove file.",blocks[idx].delete());
    }
    assertTrue("Corrupted replicas not handled properly.",util.checkFiles(fs,"/srcdat"));
    util.cleanup(fs,"/srcdat");
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestFileCreation

APIUtilityVerifier BranchVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Create a file, write something, hflush but not close.
 * Then change lease period and wait for lease recovery.
 * Finally, read the block directly from each Datanode and verify the content.
 */
@Test public void testLeaseExpireHardLimit() throws Exception {
  System.out.println("testLeaseExpireHardLimit start");
  final long leasePeriod=1000;
  final int DATANODE_NUM=3;
  Configuration conf=new HdfsConfiguration();
  conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,1000);
  conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY,1);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_NUM).build();
  DistributedFileSystem dfs=null;
  try {
    cluster.waitActive();
    dfs=cluster.getFileSystem();
    final String f=DIR + "foo";
    final Path fpath=new Path(f);
    HdfsDataOutputStream out=create(dfs,fpath,DATANODE_NUM);
    out.write("something".getBytes());
    out.hflush();
    int actualRepl=out.getCurrentBlockReplication();
    assertTrue(f + " should be replicated to " + DATANODE_NUM+ " datanodes.",actualRepl == DATANODE_NUM);
    cluster.setLeasePeriod(leasePeriod,leasePeriod);
    try {
      Thread.sleep(5 * leasePeriod);
    }
 catch (    InterruptedException e) {
    }
    LocatedBlocks locations=dfs.dfs.getNamenode().getBlockLocations(f,0,Long.MAX_VALUE);
    assertEquals(1,locations.locatedBlockCount());
    LocatedBlock locatedblock=locations.getLocatedBlocks().get(0);
    int successcount=0;
    for (    DatanodeInfo datanodeinfo : locatedblock.getLocations()) {
      DataNode datanode=cluster.getDataNode(datanodeinfo.getIpcPort());
      ExtendedBlock blk=locatedblock.getBlock();
      Block b=DataNodeTestUtils.getFSDataset(datanode).getStoredBlock(blk.getBlockPoolId(),blk.getBlockId());
      final File blockfile=DataNodeTestUtils.getFile(datanode,blk.getBlockPoolId(),b.getBlockId());
      System.out.println("blockfile=" + blockfile);
      if (blockfile != null) {
        BufferedReader in=new BufferedReader(new FileReader(blockfile));
        assertEquals("something",in.readLine());
        in.close();
        successcount++;
      }
    }
    System.out.println("successcount=" + successcount);
    assertTrue(successcount > 0);
  }
  finally {
    IOUtils.closeStream(dfs);
    cluster.shutdown();
  }
  System.out.println("testLeaseExpireHardLimit successful");
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Test that file leases are persisted across namenode restarts.
 */
@Test public void testFileCreationNamenodeRestart() throws IOException {
  Configuration conf=new HdfsConfiguration();
  final int MAX_IDLE_TIME=2000;
  conf.setInt("ipc.client.connection.maxidletime",MAX_IDLE_TIME);
  conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,1000);
  conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY,1);
  if (simulatedStorage) {
    SimulatedFSDataset.setFactory(conf);
  }
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  DistributedFileSystem fs=null;
  try {
    cluster.waitActive();
    fs=cluster.getFileSystem();
    final int nnport=cluster.getNameNodePort();
    Path file1=new Path("/filestatus.dat");
    HdfsDataOutputStream stm=create(fs,file1,1);
    System.out.println("testFileCreationNamenodeRestart: " + "Created file " + file1);
    assertEquals(file1 + " should be replicated to 1 datanode.",1,stm.getCurrentBlockReplication());
    writeFile(stm,numBlocks * blockSize);
    stm.hflush();
    assertEquals(file1 + " should still be replicated to 1 datanode.",1,stm.getCurrentBlockReplication());
    Path fileRenamed=new Path("/filestatusRenamed.dat");
    fs.rename(file1,fileRenamed);
    System.out.println("testFileCreationNamenodeRestart: " + "Renamed file " + file1 + " to "+ fileRenamed);
    file1=fileRenamed;
    Path file2=new Path("/filestatus2.dat");
    FSDataOutputStream stm2=createFile(fs,file2,1);
    System.out.println("testFileCreationNamenodeRestart: " + "Created file " + file2);
    Path file3=new Path("/user/home/fullpath.dat");
    FSDataOutputStream stm3=createFile(fs,file3,1);
    System.out.println("testFileCreationNamenodeRestart: " + "Created file " + file3);
    Path file4=new Path("/user/home/fullpath4.dat");
    FSDataOutputStream stm4=createFile(fs,file4,1);
    System.out.println("testFileCreationNamenodeRestart: " + "Created file " + file4);
    fs.mkdirs(new Path("/bin"));
    fs.rename(new Path("/user/home"),new Path("/bin"));
    Path file3new=new Path("/bin/home/fullpath.dat");
    System.out.println("testFileCreationNamenodeRestart: " + "Renamed file " + file3 + " to "+ file3new);
    Path file4new=new Path("/bin/home/fullpath4.dat");
    System.out.println("testFileCreationNamenodeRestart: " + "Renamed file " + file4 + " to "+ file4new);
    cluster.shutdown();
    try {
      Thread.sleep(2 * MAX_IDLE_TIME);
    }
 catch (    InterruptedException e) {
    }
    cluster=new MiniDFSCluster.Builder(conf).nameNodePort(nnport).format(false).build();
    cluster.waitActive();
    cluster.shutdown();
    try {
      Thread.sleep(5000);
    }
 catch (    InterruptedException e) {
    }
    cluster=new MiniDFSCluster.Builder(conf).nameNodePort(nnport).format(false).build();
    cluster.waitActive();
    fs=cluster.getFileSystem();
    DFSOutputStream dfstream=(DFSOutputStream)(stm.getWrappedStream());
    dfstream.setTestFilename(file1.toString());
    dfstream=(DFSOutputStream)(stm3.getWrappedStream());
    dfstream.setTestFilename(file3new.toString());
    dfstream=(DFSOutputStream)(stm4.getWrappedStream());
    dfstream.setTestFilename(file4new.toString());
    byte[] buffer=AppendTestUtil.randomBytes(seed,1);
    stm.write(buffer);
    stm.close();
    stm2.write(buffer);
    stm2.close();
    stm3.close();
    stm4.close();
    DFSClient client=fs.dfs;
    LocatedBlocks locations=client.getNamenode().getBlockLocations(file1.toString(),0,Long.MAX_VALUE);
    System.out.println("locations = " + locations.locatedBlockCount());
    assertTrue("Error blocks were not cleaned up for file " + file1,locations.locatedBlockCount() == 3);
    locations=client.getNamenode().getBlockLocations(file2.toString(),0,Long.MAX_VALUE);
    System.out.println("locations = " + locations.locatedBlockCount());
    assertTrue("Error blocks were not cleaned up for file " + file2,locations.locatedBlockCount() == 1);
  }
  finally {
    IOUtils.closeStream(fs);
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Test deleteOnExit
 */
@Test public void testDeleteOnExit() throws IOException {
  Configuration conf=new HdfsConfiguration();
  if (simulatedStorage) {
    SimulatedFSDataset.setFactory(conf);
  }
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  FileSystem fs=cluster.getFileSystem();
  FileSystem localfs=FileSystem.getLocal(conf);
  try {
    Path file1=new Path("filestatus.dat");
    Path file2=new Path("filestatus2.dat");
    Path file3=new Path("filestatus3.dat");
    FSDataOutputStream stm1=createFile(fs,file1,1);
    FSDataOutputStream stm2=createFile(fs,file2,1);
    FSDataOutputStream stm3=createFile(localfs,file3,1);
    System.out.println("DeleteOnExit: Created files.");
    writeFile(stm1);
    writeFile(stm3);
    stm1.close();
    stm2.close();
    stm3.close();
    fs.deleteOnExit(file1);
    fs.deleteOnExit(file2);
    localfs.deleteOnExit(file3);
    fs.close();
    localfs.close();
    fs=null;
    localfs=null;
    fs=cluster.getFileSystem();
    localfs=FileSystem.getLocal(conf);
    assertTrue(file1 + " still exists inspite of deletOnExit set.",!fs.exists(file1));
    assertTrue(file2 + " still exists inspite of deletOnExit set.",!fs.exists(file2));
    assertTrue(file3 + " still exists inspite of deletOnExit set.",!localfs.exists(file3));
    System.out.println("DeleteOnExit successful.");
  }
  finally {
    IOUtils.closeStream(fs);
    IOUtils.closeStream(localfs);
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestGetBlocks

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier 

/** 
 * test getBlocks 
 */
@Test public void testGetBlocks() throws Exception {
  final Configuration CONF=new HdfsConfiguration();
  final short REPLICATION_FACTOR=(short)2;
  final int DEFAULT_BLOCK_SIZE=1024;
  CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,DEFAULT_BLOCK_SIZE);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(CONF).numDataNodes(REPLICATION_FACTOR).build();
  try {
    cluster.waitActive();
    long fileLen=2 * DEFAULT_BLOCK_SIZE;
    DFSTestUtil.createFile(cluster.getFileSystem(),new Path("/tmp.txt"),fileLen,REPLICATION_FACTOR,0L);
    List locatedBlocks;
    DatanodeInfo[] dataNodes=null;
    boolean notWritten;
    do {
      final DFSClient dfsclient=new DFSClient(NameNode.getAddress(CONF),CONF);
      locatedBlocks=dfsclient.getNamenode().getBlockLocations("/tmp.txt",0,fileLen).getLocatedBlocks();
      assertEquals(2,locatedBlocks.size());
      notWritten=false;
      for (int i=0; i < 2; i++) {
        dataNodes=locatedBlocks.get(i).getLocations();
        if (dataNodes.length != REPLICATION_FACTOR) {
          notWritten=true;
          try {
            Thread.sleep(10);
          }
 catch (          InterruptedException e) {
          }
          break;
        }
      }
    }
 while (notWritten);
    InetSocketAddress addr=new InetSocketAddress("localhost",cluster.getNameNodePort());
    NamenodeProtocol namenode=NameNodeProxies.createProxy(CONF,NameNode.getUri(addr),NamenodeProtocol.class).getProxy();
    BlockWithLocations[] locs;
    locs=namenode.getBlocks(dataNodes[0],fileLen).getBlocks();
    assertEquals(locs.length,2);
    assertEquals(locs[0].getStorageIDs().length,2);
    assertEquals(locs[1].getStorageIDs().length,2);
    locs=namenode.getBlocks(dataNodes[0],DEFAULT_BLOCK_SIZE).getBlocks();
    assertEquals(locs.length,1);
    assertEquals(locs[0].getStorageIDs().length,2);
    locs=namenode.getBlocks(dataNodes[0],1).getBlocks();
    assertEquals(locs.length,1);
    assertEquals(locs[0].getStorageIDs().length,2);
    getBlocksWithException(namenode,dataNodes[0],0);
    getBlocksWithException(namenode,dataNodes[0],-1);
    DatanodeInfo info=DFSTestUtil.getDatanodeInfo("1.2.3.4");
    getBlocksWithException(namenode,info,2);
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier IterativeVerifier EqualityVerifier 

@Test public void testBlockKey(){
  Map map=new HashMap();
  final Random RAN=new Random();
  final long seed=RAN.nextLong();
  System.out.println("seed=" + seed);
  RAN.setSeed(seed);
  long[] blkids=new long[10];
  for (int i=0; i < blkids.length; i++) {
    blkids[i]=1000L + RAN.nextInt(100000);
    map.put(new Block(blkids[i],0,blkids[i]),blkids[i]);
  }
  System.out.println("map=" + map.toString().replace(",","\n  "));
  for (int i=0; i < blkids.length; i++) {
    Block b=new Block(blkids[i],0,GenerationStamp.GRANDFATHER_GENERATION_STAMP);
    Long v=map.get(b);
    System.out.println(b + " => " + v);
    assertEquals(blkids[i],v.longValue());
  }
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test if the datanodes returned by{@link ClientProtocol#getBlockLocations(String,long,long)} is correct
 * when stale nodes checking is enabled. Also test during the scenario when 1)
 * stale nodes checking is enabled, 2) a writing is going on, 3) a datanode
 * becomes stale happen simultaneously
 * @throws Exception
 */
@Test public void testReadSelectNonStaleDatanode() throws Exception {
  HdfsConfiguration conf=new HdfsConfiguration();
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY,true);
  long staleInterval=30 * 1000 * 60;
  conf.setLong(DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY,staleInterval);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).racks(racks).build();
  cluster.waitActive();
  InetSocketAddress addr=new InetSocketAddress("localhost",cluster.getNameNodePort());
  DFSClient client=new DFSClient(addr,conf);
  List nodeInfoList=cluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getDatanodeListForReport(DatanodeReportType.LIVE);
  assertEquals("Unexpected number of datanodes",numDatanodes,nodeInfoList.size());
  FileSystem fileSys=cluster.getFileSystem();
  FSDataOutputStream stm=null;
  try {
    final Path fileName=new Path("/file1");
    stm=fileSys.create(fileName,true,fileSys.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY,4096),(short)3,blockSize);
    stm.write(new byte[(blockSize * 3) / 2]);
    stm.hflush();
    LocatedBlocks blocks=client.getNamenode().getBlockLocations(fileName.toString(),0,blockSize);
    DatanodeInfo[] nodes=blocks.get(0).getLocations();
    assertEquals(nodes.length,3);
    DataNode staleNode=null;
    DatanodeDescriptor staleNodeInfo=null;
    staleNode=this.stopDataNodeHeartbeat(cluster,nodes[0].getHostName());
    assertNotNull(staleNode);
    staleNodeInfo=cluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getDatanode(staleNode.getDatanodeId());
    staleNodeInfo.setLastUpdate(Time.now() - staleInterval - 1);
    LocatedBlocks blocksAfterStale=client.getNamenode().getBlockLocations(fileName.toString(),0,blockSize);
    DatanodeInfo[] nodesAfterStale=blocksAfterStale.get(0).getLocations();
    assertEquals(nodesAfterStale.length,3);
    assertEquals(nodesAfterStale[2].getHostName(),nodes[0].getHostName());
    DataNodeTestUtils.setHeartbeatsDisabledForTests(staleNode,false);
    staleNodeInfo.setLastUpdate(Time.now());
    LocatedBlock lastBlock=client.getLocatedBlocks(fileName.toString(),0,Long.MAX_VALUE).getLastLocatedBlock();
    nodes=lastBlock.getLocations();
    assertEquals(nodes.length,3);
    staleNode=this.stopDataNodeHeartbeat(cluster,nodes[0].getHostName());
    assertNotNull(staleNode);
    cluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getDatanode(staleNode.getDatanodeId()).setLastUpdate(Time.now() - staleInterval - 1);
    LocatedBlock lastBlockAfterStale=client.getLocatedBlocks(fileName.toString(),0,Long.MAX_VALUE).getLastLocatedBlock();
    nodesAfterStale=lastBlockAfterStale.getLocations();
    assertEquals(nodesAfterStale.length,3);
    assertEquals(nodesAfterStale[2].getHostName(),nodes[0].getHostName());
  }
  finally {
    if (stm != null) {
      stm.close();
    }
    if (client != null) {
      client.close();
    }
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestHDFSServerPorts

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Verify datanode port usage.
 */
@Test public void testDataNodePorts() throws Exception {
  NameNode nn=null;
  try {
    nn=startNameNode();
    Configuration conf2=new HdfsConfiguration(config);
    conf2.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,new File(hdfsDir,"data").getPath());
    conf2.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY,FileSystem.getDefaultUri(config).getAuthority());
    conf2.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY,THIS_HOST);
    boolean started=canStartDataNode(conf2);
    assertFalse(started);
    conf2.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY,THIS_HOST);
    conf2.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY,config.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY));
    started=canStartDataNode(conf2);
    assertFalse(started);
    conf2.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY,THIS_HOST);
    conf2.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY,THIS_HOST);
    conf2.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY,THIS_HOST);
    started=canStartDataNode(conf2);
    assertTrue(started);
  }
  finally {
    stopNameNode(nn);
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Verify BackupNode port usage.
 */
@Test public void testBackupNodePorts() throws Exception {
  NameNode nn=null;
  try {
    nn=startNameNode();
    Configuration backup_config=new HdfsConfiguration(config);
    backup_config.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY,THIS_HOST);
    backup_config.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,backup_config.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY));
    LOG.info("= Starting 1 on: " + backup_config.get(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY));
    assertFalse("Backup started on same port as Namenode",canStartBackupNode(backup_config));
    backup_config.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,THIS_HOST);
    LOG.info("= Starting 2 on: " + backup_config.get(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY));
    boolean started=canStartBackupNode(backup_config);
    assertTrue("Backup Namenode should've started",started);
  }
  finally {
    stopNameNode(nn);
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Verify secondary namenode port usage.
 */
@Test public void testSecondaryNodePorts() throws Exception {
  NameNode nn=null;
  try {
    nn=startNameNode();
    Configuration conf2=new HdfsConfiguration(config);
    conf2.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,config.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY));
    LOG.info("= Starting 1 on: " + conf2.get(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY));
    boolean started=canStartSecondaryNode(conf2);
    assertFalse(started);
    conf2.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,THIS_HOST);
    LOG.info("= Starting 2 on: " + conf2.get(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY));
    started=canStartSecondaryNode(conf2);
    assertTrue(started);
  }
  finally {
    stopNameNode(nn);
  }
}

Class: org.apache.hadoop.hdfs.TestHdfsAdmin

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that we can set and clear quotas via {@link HdfsAdmin}.
 */
@Test public void testHdfsAdminSetQuota() throws Exception {
  HdfsAdmin dfsAdmin=new HdfsAdmin(FileSystem.getDefaultUri(conf),conf);
  FileSystem fs=null;
  try {
    fs=FileSystem.get(conf);
    assertTrue(fs.mkdirs(TEST_PATH));
    assertEquals(-1,fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(-1,fs.getContentSummary(TEST_PATH).getSpaceQuota());
    dfsAdmin.setSpaceQuota(TEST_PATH,10);
    assertEquals(-1,fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(10,fs.getContentSummary(TEST_PATH).getSpaceQuota());
    dfsAdmin.setQuota(TEST_PATH,10);
    assertEquals(10,fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(10,fs.getContentSummary(TEST_PATH).getSpaceQuota());
    dfsAdmin.clearSpaceQuota(TEST_PATH);
    assertEquals(10,fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(-1,fs.getContentSummary(TEST_PATH).getSpaceQuota());
    dfsAdmin.clearQuota(TEST_PATH);
    assertEquals(-1,fs.getContentSummary(TEST_PATH).getQuota());
    assertEquals(-1,fs.getContentSummary(TEST_PATH).getSpaceQuota());
  }
  finally {
    if (fs != null) {
      fs.close();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestIsMethodSupported

APIUtilityVerifier BooleanVerifier 

@Test public void testClientDatanodeProtocol() throws IOException {
  ClientDatanodeProtocolTranslatorPB translator=new ClientDatanodeProtocolTranslatorPB(nnAddress,UserGroupInformation.getCurrentUser(),conf,NetUtils.getDefaultSocketFactory(conf));
  assertFalse(translator.isMethodSupported("refreshNamenodes"));
  translator=new ClientDatanodeProtocolTranslatorPB(dnAddress,UserGroupInformation.getCurrentUser(),conf,NetUtils.getDefaultSocketFactory(conf));
  assertTrue(translator.isMethodSupported("refreshNamenodes"));
}

APIUtilityVerifier BooleanVerifier 

@Test public void tesJournalProtocol() throws IOException {
  JournalProtocolTranslatorPB translator=(JournalProtocolTranslatorPB)NameNodeProxies.createNonHAProxy(conf,nnAddress,JournalProtocol.class,UserGroupInformation.getCurrentUser(),true).getProxy();
  assertFalse(translator.isMethodSupported("startLogSegment"));
}

APIUtilityVerifier BooleanVerifier 

@Test public void testRefreshCallQueueProtocol() throws IOException {
  RefreshCallQueueProtocolClientSideTranslatorPB translator=(RefreshCallQueueProtocolClientSideTranslatorPB)NameNodeProxies.createNonHAProxy(conf,nnAddress,RefreshCallQueueProtocol.class,UserGroupInformation.getCurrentUser(),true).getProxy();
  assertTrue(translator.isMethodSupported("refreshCallQueue"));
}

APIUtilityVerifier BooleanVerifier 

@Test public void testInterDatanodeProtocol() throws IOException {
  InterDatanodeProtocolTranslatorPB translator=new InterDatanodeProtocolTranslatorPB(nnAddress,UserGroupInformation.getCurrentUser(),conf,NetUtils.getDefaultSocketFactory(conf),0);
  assertFalse(translator.isMethodSupported("initReplicaRecovery"));
  translator=new InterDatanodeProtocolTranslatorPB(dnAddress,UserGroupInformation.getCurrentUser(),conf,NetUtils.getDefaultSocketFactory(conf),0);
  assertTrue(translator.isMethodSupported("initReplicaRecovery"));
}

APIUtilityVerifier BooleanVerifier 

@Test public void testRefreshUserMappingsProtocol() throws IOException {
  RefreshUserMappingsProtocolClientSideTranslatorPB translator=(RefreshUserMappingsProtocolClientSideTranslatorPB)NameNodeProxies.createNonHAProxy(conf,nnAddress,RefreshUserMappingsProtocol.class,UserGroupInformation.getCurrentUser(),true).getProxy();
  assertTrue(translator.isMethodSupported("refreshUserToGroupsMappings"));
}

APIUtilityVerifier BooleanVerifier 

@Test public void testRefreshAuthorizationPolicyProtocol() throws IOException {
  RefreshAuthorizationPolicyProtocolClientSideTranslatorPB translator=(RefreshAuthorizationPolicyProtocolClientSideTranslatorPB)NameNodeProxies.createNonHAProxy(conf,nnAddress,RefreshAuthorizationPolicyProtocol.class,UserGroupInformation.getCurrentUser(),true).getProxy();
  assertTrue(translator.isMethodSupported("refreshServiceAcl"));
}

APIUtilityVerifier BooleanVerifier 

@Test public void testNamenodeProtocol() throws IOException {
  NamenodeProtocol np=NameNodeProxies.createNonHAProxy(conf,nnAddress,NamenodeProtocol.class,UserGroupInformation.getCurrentUser(),true).getProxy();
  boolean exists=RpcClientUtil.isMethodSupported(np,NamenodeProtocolPB.class,RPC.RpcKind.RPC_PROTOCOL_BUFFER,RPC.getProtocolVersion(NamenodeProtocolPB.class),"rollEditLog");
  assertTrue(exists);
  exists=RpcClientUtil.isMethodSupported(np,NamenodeProtocolPB.class,RPC.RpcKind.RPC_PROTOCOL_BUFFER,RPC.getProtocolVersion(NamenodeProtocolPB.class),"bogusMethod");
  assertFalse(exists);
}

APIUtilityVerifier BooleanVerifier 

@Test public void testGetUserMappingsProtocol() throws IOException {
  GetUserMappingsProtocolClientSideTranslatorPB translator=(GetUserMappingsProtocolClientSideTranslatorPB)NameNodeProxies.createNonHAProxy(conf,nnAddress,GetUserMappingsProtocol.class,UserGroupInformation.getCurrentUser(),true).getProxy();
  assertTrue(translator.isMethodSupported("getGroupsForUser"));
}

Class: org.apache.hadoop.hdfs.TestLease

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@SuppressWarnings("unchecked") @Test public void testFactory() throws Exception {
  final String[] groups=new String[]{"supergroup"};
  final UserGroupInformation[] ugi=new UserGroupInformation[3];
  for (int i=0; i < ugi.length; i++) {
    ugi[i]=UserGroupInformation.createUserForTesting("user" + i,groups);
  }
  Mockito.doReturn(new HdfsFileStatus(0,false,1,1024,0,0,new FsPermission((short)777),"owner","group",new byte[0],new byte[0],1010,0,null)).when(mcp).getFileInfo(anyString());
  Mockito.doReturn(new HdfsFileStatus(0,false,1,1024,0,0,new FsPermission((short)777),"owner","group",new byte[0],new byte[0],1010,0,null)).when(mcp).create(anyString(),(FsPermission)anyObject(),anyString(),(EnumSetWritable)anyObject(),anyBoolean(),anyShort(),anyLong(),(List)anyList());
  final Configuration conf=new Configuration();
  final DFSClient c1=createDFSClientAs(ugi[0],conf);
  FSDataOutputStream out1=createFsOut(c1,"/out1");
  final DFSClient c2=createDFSClientAs(ugi[0],conf);
  FSDataOutputStream out2=createFsOut(c2,"/out2");
  Assert.assertEquals(c1.getLeaseRenewer(),c2.getLeaseRenewer());
  final DFSClient c3=createDFSClientAs(ugi[1],conf);
  FSDataOutputStream out3=createFsOut(c3,"/out3");
  Assert.assertTrue(c1.getLeaseRenewer() != c3.getLeaseRenewer());
  final DFSClient c4=createDFSClientAs(ugi[1],conf);
  FSDataOutputStream out4=createFsOut(c4,"/out4");
  Assert.assertEquals(c3.getLeaseRenewer(),c4.getLeaseRenewer());
  final DFSClient c5=createDFSClientAs(ugi[2],conf);
  FSDataOutputStream out5=createFsOut(c5,"/out5");
  Assert.assertTrue(c1.getLeaseRenewer() != c5.getLeaseRenewer());
  Assert.assertTrue(c3.getLeaseRenewer() != c5.getLeaseRenewer());
}

APIUtilityVerifier BranchVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

@Test public void testLeaseAbort() throws Exception {
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  try {
    cluster.waitActive();
    NamenodeProtocols preSpyNN=cluster.getNameNodeRpc();
    NamenodeProtocols spyNN=spy(preSpyNN);
    DFSClient dfs=new DFSClient(null,spyNN,conf,null);
    byte[] buf=new byte[1024];
    FSDataOutputStream c_out=createFsOut(dfs,dirString + "c");
    c_out.write(buf,0,1024);
    c_out.close();
    DFSInputStream c_in=dfs.open(dirString + "c");
    FSDataOutputStream d_out=createFsOut(dfs,dirString + "d");
    doThrow(new RemoteException(InvalidToken.class.getName(),"Your token is worthless")).when(spyNN).renewLease(anyString());
    LeaseRenewer originalRenewer=dfs.getLeaseRenewer();
    dfs.lastLeaseRenewal=Time.now() - HdfsConstants.LEASE_SOFTLIMIT_PERIOD - 1000;
    try {
      dfs.renewLease();
    }
 catch (    IOException e) {
    }
    try {
      d_out.write(buf,0,1024);
      LOG.info("Write worked beyond the soft limit as expected.");
    }
 catch (    IOException e) {
      Assert.fail("Write failed.");
    }
    dfs.lastLeaseRenewal=Time.now() - HdfsConstants.LEASE_HARDLIMIT_PERIOD - 1000;
    dfs.renewLease();
    try {
      d_out.write(buf,0,1024);
      d_out.close();
      Assert.fail("Write did not fail even after the fatal lease renewal failure");
    }
 catch (    IOException e) {
      LOG.info("Write failed as expected. ",e);
    }
    Thread.sleep(1000);
    Assert.assertTrue(originalRenewer.isEmpty());
    doNothing().when(spyNN).renewLease(anyString());
    try {
      int num=c_in.read(buf,0,1);
      if (num != 1) {
        Assert.fail("Failed to read 1 byte");
      }
      c_in.close();
    }
 catch (    IOException e) {
      LOG.error("Read failed with ",e);
      Assert.fail("Read after lease renewal failure failed");
    }
    try {
      c_out=createFsOut(dfs,dirString + "c");
      c_out.write(buf,0,1024);
      c_out.close();
    }
 catch (    IOException e) {
      LOG.error("Write failed with ",e);
      Assert.fail("Write failed");
    }
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestLeaseRecovery

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * The following test first creates a file with a few blocks.
 * It randomly truncates the replica of the last block stored in each datanode.
 * Finally, it triggers block synchronization to synchronize all stored block.
 */
@Test public void testBlockSynchronization() throws Exception {
  final int ORG_FILE_SIZE=3000;
  Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,BLOCK_SIZE);
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(5).build();
    cluster.waitActive();
    DistributedFileSystem dfs=cluster.getFileSystem();
    String filestr="/foo";
    Path filepath=new Path(filestr);
    DFSTestUtil.createFile(dfs,filepath,ORG_FILE_SIZE,REPLICATION_NUM,0L);
    assertTrue(dfs.exists(filepath));
    DFSTestUtil.waitReplication(dfs,filepath,REPLICATION_NUM);
    LocatedBlock locatedblock=TestInterDatanodeProtocol.getLastLocatedBlock(dfs.dfs.getNamenode(),filestr);
    DatanodeInfo[] datanodeinfos=locatedblock.getLocations();
    assertEquals(REPLICATION_NUM,datanodeinfos.length);
    DataNode[] datanodes=new DataNode[REPLICATION_NUM];
    for (int i=0; i < REPLICATION_NUM; i++) {
      datanodes[i]=cluster.getDataNode(datanodeinfos[i].getIpcPort());
      assertTrue(datanodes[i] != null);
    }
    ExtendedBlock lastblock=locatedblock.getBlock();
    DataNode.LOG.info("newblocks=" + lastblock);
    for (int i=0; i < REPLICATION_NUM; i++) {
      checkMetaInfo(lastblock,datanodes[i]);
    }
    DataNode.LOG.info("dfs.dfs.clientName=" + dfs.dfs.clientName);
    cluster.getNameNodeRpc().append(filestr,dfs.dfs.clientName);
    waitLeaseRecovery(cluster);
    Block[] updatedmetainfo=new Block[REPLICATION_NUM];
    long oldSize=lastblock.getNumBytes();
    lastblock=TestInterDatanodeProtocol.getLastLocatedBlock(dfs.dfs.getNamenode(),filestr).getBlock();
    long currentGS=lastblock.getGenerationStamp();
    for (int i=0; i < REPLICATION_NUM; i++) {
      updatedmetainfo[i]=DataNodeTestUtils.getFSDataset(datanodes[i]).getStoredBlock(lastblock.getBlockPoolId(),lastblock.getBlockId());
      assertEquals(lastblock.getBlockId(),updatedmetainfo[i].getBlockId());
      assertEquals(oldSize,updatedmetainfo[i].getNumBytes());
      assertEquals(currentGS,updatedmetainfo[i].getGenerationStamp());
    }
    System.out.println("Testing that lease recovery cannot happen during safemode.");
    filestr="/foo.safemode";
    filepath=new Path(filestr);
    dfs.create(filepath,(short)1);
    cluster.getNameNodeRpc().setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER,false);
    assertTrue(dfs.dfs.exists(filestr));
    DFSTestUtil.waitReplication(dfs,filepath,(short)1);
    waitLeaseRecovery(cluster);
    LeaseManager lm=NameNodeAdapter.getLeaseManager(cluster.getNamesystem());
    assertTrue("Found " + lm.countLease() + " lease, expected 1",lm.countLease() == 1);
    cluster.getNameNodeRpc().setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,false);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Block Recovery when the meta file not having crcs for all chunks in block
 * file
 */
@Test public void testBlockRecoveryWithLessMetafile() throws Exception {
  Configuration conf=new Configuration();
  conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY,UserGroupInformation.getCurrentUser().getShortUserName());
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  Path file=new Path("/testRecoveryFile");
  DistributedFileSystem dfs=cluster.getFileSystem();
  FSDataOutputStream out=dfs.create(file);
  int count=0;
  while (count < 2 * 1024 * 1024) {
    out.writeBytes("Data");
    count+=4;
  }
  out.hsync();
  ((DFSOutputStream)out.getWrappedStream()).abort();
  LocatedBlocks locations=cluster.getNameNodeRpc().getBlockLocations(file.toString(),0,count);
  ExtendedBlock block=locations.get(0).getBlock();
  DataNode dn=cluster.getDataNodes().get(0);
  BlockLocalPathInfo localPathInfo=dn.getBlockLocalPathInfo(block,null);
  File metafile=new File(localPathInfo.getMetaPath());
  assertTrue(metafile.exists());
  RandomAccessFile raf=new RandomAccessFile(metafile,"rw");
  raf.setLength(metafile.length() - 20);
  raf.close();
  DataNodeProperties dnProp=cluster.stopDataNode(0);
  cluster.restartDataNode(dnProp,true);
  DistributedFileSystem newdfs=(DistributedFileSystem)FileSystem.newInstance(cluster.getConfiguration(0));
  count=0;
  while (++count < 10 && !newdfs.recoverLease(file)) {
    Thread.sleep(1000);
  }
  assertTrue("File should be closed",newdfs.recoverLease(file));
}

Class: org.apache.hadoop.hdfs.TestLeaseRecovery2

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * This test makes the client does not renew its lease and also
 * set the hard lease expiration period to be short 1s. Thus triggering
 * lease expiration to happen while the client is still alive.
 * The test makes sure that the lease recovery completes and the client
 * fails if it continues to write to the file.
 * @throws Exception
 */
@Test public void testHardLeaseRecovery() throws Exception {
  String filestr="/hardLeaseRecovery";
  AppendTestUtil.LOG.info("filestr=" + filestr);
  Path filepath=new Path(filestr);
  FSDataOutputStream stm=dfs.create(filepath,true,BUF_SIZE,REPLICATION_NUM,BLOCK_SIZE);
  assertTrue(dfs.dfs.exists(filestr));
  int size=AppendTestUtil.nextInt(FILE_SIZE);
  AppendTestUtil.LOG.info("size=" + size);
  stm.write(buffer,0,size);
  AppendTestUtil.LOG.info("hflush");
  stm.hflush();
  AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()");
  dfs.dfs.getLeaseRenewer().interruptAndJoin();
  cluster.setLeasePeriod(LONG_LEASE_PERIOD,SHORT_LEASE_PERIOD);
  LocatedBlocks locatedBlocks;
  do {
    Thread.sleep(SHORT_LEASE_PERIOD);
    locatedBlocks=dfs.dfs.getLocatedBlocks(filestr,0L,size);
  }
 while (locatedBlocks.isUnderConstruction());
  assertEquals(size,locatedBlocks.getFileLength());
  try {
    stm.write('b');
    stm.close();
    fail("Writer thread should have been killed");
  }
 catch (  IOException e) {
    e.printStackTrace();
  }
  AppendTestUtil.LOG.info("File size is good. Now validating sizes from datanodes...");
  AppendTestUtil.checkFullFile(dfs,filepath,size,buffer,filestr);
}

APIUtilityVerifier IterativeVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * This test makes the client does not renew its lease and also
 * set the soft lease expiration period to be short 1s. Thus triggering
 * soft lease expiration to happen immediately by having another client
 * trying to create the same file.
 * The test makes sure that the lease recovery completes.
 * @throws Exception
 */
@Test public void testSoftLeaseRecovery() throws Exception {
  Map u2g_map=new HashMap(1);
  u2g_map.put(fakeUsername,new String[]{fakeGroup});
  DFSTestUtil.updateConfWithFakeGroupMapping(conf,u2g_map);
  cluster.setLeasePeriod(HdfsConstants.LEASE_SOFTLIMIT_PERIOD,HdfsConstants.LEASE_HARDLIMIT_PERIOD);
  String filestr="/foo" + AppendTestUtil.nextInt();
  AppendTestUtil.LOG.info("filestr=" + filestr);
  Path filepath=new Path(filestr);
  FSDataOutputStream stm=dfs.create(filepath,true,BUF_SIZE,REPLICATION_NUM,BLOCK_SIZE);
  assertTrue(dfs.dfs.exists(filestr));
  int size=AppendTestUtil.nextInt(FILE_SIZE);
  AppendTestUtil.LOG.info("size=" + size);
  stm.write(buffer,0,size);
  AppendTestUtil.LOG.info("hflush");
  stm.hflush();
  AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()");
  dfs.dfs.getLeaseRenewer().interruptAndJoin();
  cluster.setLeasePeriod(SHORT_LEASE_PERIOD,LONG_LEASE_PERIOD);
{
    UserGroupInformation ugi=UserGroupInformation.createUserForTesting(fakeUsername,new String[]{fakeGroup});
    FileSystem dfs2=DFSTestUtil.getFileSystemAs(ugi,conf);
    boolean done=false;
    for (int i=0; i < 10 && !done; i++) {
      AppendTestUtil.LOG.info("i=" + i);
      try {
        dfs2.create(filepath,false,BUF_SIZE,REPLICATION_NUM,BLOCK_SIZE);
        fail("Creation of an existing file should never succeed.");
      }
 catch (      FileAlreadyExistsException ex) {
        done=true;
      }
catch (      AlreadyBeingCreatedException ex) {
        AppendTestUtil.LOG.info("GOOD! got " + ex.getMessage());
      }
catch (      IOException ioe) {
        AppendTestUtil.LOG.warn("UNEXPECTED IOException",ioe);
      }
      if (!done) {
        AppendTestUtil.LOG.info("sleep " + 5000 + "ms");
        try {
          Thread.sleep(5000);
        }
 catch (        InterruptedException e) {
        }
      }
    }
    assertTrue(done);
  }
  AppendTestUtil.LOG.info("Lease for file " + filepath + " is recovered. "+ "Validating its contents now...");
  long fileSize=dfs.getFileStatus(filepath).getLen();
  assertTrue("File should be " + size + " bytes, but is actually "+ " found to be "+ fileSize+ " bytes",fileSize == size);
  AppendTestUtil.LOG.info("File size is good. " + "Now validating data and sizes from datanodes...");
  AppendTestUtil.checkFullFile(dfs,filepath,size,buffer,filestr);
}

Class: org.apache.hadoop.hdfs.TestLeaseRenewer

APIUtilityVerifier IdentityVerifier 

@Test public void testInstanceSharing() throws IOException {
  LeaseRenewer lr=LeaseRenewer.getInstance(FAKE_AUTHORITY,FAKE_UGI_A,MOCK_DFSCLIENT);
  LeaseRenewer lr2=LeaseRenewer.getInstance(FAKE_AUTHORITY,FAKE_UGI_A,MOCK_DFSCLIENT);
  Assert.assertSame(lr,lr2);
  LeaseRenewer lr3=LeaseRenewer.getInstance(FAKE_AUTHORITY,FAKE_UGI_B,MOCK_DFSCLIENT);
  Assert.assertNotSame(lr,lr3);
  LeaseRenewer lr4=LeaseRenewer.getInstance("someOtherAuthority",FAKE_UGI_B,MOCK_DFSCLIENT);
  Assert.assertNotSame(lr,lr4);
  Assert.assertNotSame(lr3,lr4);
}

Class: org.apache.hadoop.hdfs.TestListFilesInFileContext

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test when input path is a file 
 */
@Test public void testFile() throws IOException {
  fc.mkdir(TEST_DIR,FsPermission.getDefault(),true);
  writeFile(fc,FILE1,FILE_LEN);
  RemoteIterator itor=fc.util().listFiles(FILE1,true);
  LocatedFileStatus stat=itor.next();
  assertFalse(itor.hasNext());
  assertTrue(stat.isFile());
  assertEquals(FILE_LEN,stat.getLen());
  assertEquals(fc.makeQualified(FILE1),stat.getPath());
  assertEquals(1,stat.getBlockLocations().length);
  itor=fc.util().listFiles(FILE1,false);
  stat=itor.next();
  assertFalse(itor.hasNext());
  assertTrue(stat.isFile());
  assertEquals(FILE_LEN,stat.getLen());
  assertEquals(fc.makeQualified(FILE1),stat.getPath());
  assertEquals(1,stat.getBlockLocations().length);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test when input path is a directory 
 */
@Test public void testDirectory() throws IOException {
  fc.mkdir(DIR1,FsPermission.getDefault(),true);
  RemoteIterator itor=fc.util().listFiles(DIR1,true);
  assertFalse(itor.hasNext());
  itor=fc.util().listFiles(DIR1,false);
  assertFalse(itor.hasNext());
  writeFile(fc,FILE2,FILE_LEN);
  itor=fc.util().listFiles(DIR1,true);
  LocatedFileStatus stat=itor.next();
  assertFalse(itor.hasNext());
  assertTrue(stat.isFile());
  assertEquals(FILE_LEN,stat.getLen());
  assertEquals(fc.makeQualified(FILE2),stat.getPath());
  assertEquals(1,stat.getBlockLocations().length);
  itor=fc.util().listFiles(DIR1,false);
  stat=itor.next();
  assertFalse(itor.hasNext());
  assertTrue(stat.isFile());
  assertEquals(FILE_LEN,stat.getLen());
  assertEquals(fc.makeQualified(FILE2),stat.getPath());
  assertEquals(1,stat.getBlockLocations().length);
  writeFile(fc,FILE1,FILE_LEN);
  writeFile(fc,FILE3,FILE_LEN);
  itor=fc.util().listFiles(TEST_DIR,true);
  stat=itor.next();
  assertTrue(stat.isFile());
  assertEquals(fc.makeQualified(FILE2),stat.getPath());
  stat=itor.next();
  assertTrue(stat.isFile());
  assertEquals(fc.makeQualified(FILE3),stat.getPath());
  stat=itor.next();
  assertTrue(stat.isFile());
  assertEquals(fc.makeQualified(FILE1),stat.getPath());
  assertFalse(itor.hasNext());
  itor=fc.util().listFiles(TEST_DIR,false);
  stat=itor.next();
  assertTrue(stat.isFile());
  assertEquals(fc.makeQualified(FILE1),stat.getPath());
  assertFalse(itor.hasNext());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test when input patch has a symbolic links as its children 
 */
@Test public void testSymbolicLinks() throws IOException {
  writeFile(fc,FILE1,FILE_LEN);
  writeFile(fc,FILE2,FILE_LEN);
  writeFile(fc,FILE3,FILE_LEN);
  Path dir4=new Path(TEST_DIR,"dir4");
  Path dir5=new Path(dir4,"dir5");
  Path file4=new Path(dir4,"file4");
  fc.createSymlink(DIR1,dir5,true);
  fc.createSymlink(FILE1,file4,true);
  RemoteIterator itor=fc.util().listFiles(dir4,true);
  LocatedFileStatus stat=itor.next();
  assertTrue(stat.isFile());
  assertEquals(fc.makeQualified(FILE2),stat.getPath());
  stat=itor.next();
  assertTrue(stat.isFile());
  assertEquals(fc.makeQualified(FILE3),stat.getPath());
  stat=itor.next();
  assertTrue(stat.isFile());
  assertEquals(fc.makeQualified(FILE1),stat.getPath());
  assertFalse(itor.hasNext());
  itor=fc.util().listFiles(dir4,false);
  stat=itor.next();
  assertTrue(stat.isFile());
  assertEquals(fc.makeQualified(FILE1),stat.getPath());
  assertFalse(itor.hasNext());
}

Class: org.apache.hadoop.hdfs.TestLocalDFS

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests get/set working directory in DFS.
 */
@Test(timeout=20000) public void testWorkingDirectory() throws IOException {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  FileSystem fileSys=cluster.getFileSystem();
  try {
    Path orig_path=fileSys.getWorkingDirectory();
    assertTrue(orig_path.isAbsolute());
    Path file1=new Path("somewhat/random.txt");
    writeFile(fileSys,file1);
    assertTrue(fileSys.exists(new Path(orig_path,file1.toString())));
    fileSys.delete(file1,true);
    Path subdir1=new Path("/somewhere");
    fileSys.setWorkingDirectory(subdir1);
    writeFile(fileSys,file1);
    cleanupFile(fileSys,new Path(subdir1,file1.toString()));
    Path subdir2=new Path("else");
    fileSys.setWorkingDirectory(subdir2);
    writeFile(fileSys,file1);
    readFile(fileSys,file1);
    cleanupFile(fileSys,new Path(new Path(subdir1,subdir2.toString()),file1.toString()));
    Path home=fileSys.makeQualified(new Path("/user/" + getUserName(fileSys)));
    Path fsHome=fileSys.getHomeDirectory();
    assertEquals(home,fsHome);
  }
  finally {
    fileSys.close();
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestMiniDFSCluster

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Bring up two clusters and assert that they are in different directories.
 * @throws Throwable on a failure
 */
@Test(timeout=100000) public void testDualClusters() throws Throwable {
  File testDataCluster2=new File(testDataPath,CLUSTER_2);
  File testDataCluster3=new File(testDataPath,CLUSTER_3);
  Configuration conf=new HdfsConfiguration();
  String c2Path=testDataCluster2.getAbsolutePath();
  conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR,c2Path);
  MiniDFSCluster cluster2=new MiniDFSCluster.Builder(conf).build();
  MiniDFSCluster cluster3=null;
  try {
    String dataDir2=cluster2.getDataDirectory();
    assertEquals(new File(c2Path + "/data"),new File(dataDir2));
    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR,testDataCluster3.getAbsolutePath());
    MiniDFSCluster.Builder builder=new MiniDFSCluster.Builder(conf);
    cluster3=builder.build();
    String dataDir3=cluster3.getDataDirectory();
    assertTrue("Clusters are bound to the same directory: " + dataDir2,!dataDir2.equals(dataDir3));
  }
  finally {
    MiniDFSCluster.shutdownCluster(cluster3);
    MiniDFSCluster.shutdownCluster(cluster2);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Verify that without system properties the cluster still comes up, provided
 * the configuration is set
 * @throws Throwable on a failure
 */
@Test(timeout=100000) public void testClusterWithoutSystemProperties() throws Throwable {
  System.clearProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA);
  Configuration conf=new HdfsConfiguration();
  File testDataCluster1=new File(testDataPath,CLUSTER_1);
  String c1Path=testDataCluster1.getAbsolutePath();
  conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR,c1Path);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  try {
    assertEquals(new File(c1Path + "/data"),new File(cluster.getDataDirectory()));
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestMissingBlocksAlert

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testMissingBlocksAlert() throws IOException, InterruptedException, MalformedObjectNameException, AttributeNotFoundException, MBeanException, ReflectionException, InstanceNotFoundException {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,0);
    conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE,10);
    int fileLen=10 * 1024;
    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,fileLen / 2);
    cluster=new MiniDFSCluster.Builder(conf).build();
    cluster.waitActive();
    final BlockManager bm=cluster.getNamesystem().getBlockManager();
    DistributedFileSystem dfs=cluster.getFileSystem();
    DFSTestUtil.createFile(dfs,new Path("/testMissingBlocksAlert/file1"),fileLen,(short)3,0);
    Path corruptFile=new Path("/testMissingBlocks/corruptFile");
    DFSTestUtil.createFile(dfs,corruptFile,fileLen,(short)3,0);
    ExtendedBlock block=DFSTestUtil.getFirstBlock(dfs,corruptFile);
    assertTrue(TestDatanodeBlockScanner.corruptReplica(block,0));
    FSDataInputStream in=dfs.open(corruptFile);
    try {
      in.readFully(new byte[fileLen]);
    }
 catch (    ChecksumException ignored) {
    }
    in.close();
    LOG.info("Waiting for missing blocks count to increase...");
    while (dfs.getMissingBlocksCount() <= 0) {
      Thread.sleep(100);
    }
    assertTrue(dfs.getMissingBlocksCount() == 1);
    assertEquals(4,dfs.getUnderReplicatedBlocksCount());
    assertEquals(3,bm.getUnderReplicatedNotMissingBlocks());
    MBeanServer mbs=ManagementFactory.getPlatformMBeanServer();
    ObjectName mxbeanName=new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo");
    Assert.assertEquals(1,(long)(Long)mbs.getAttribute(mxbeanName,"NumberOfMissingBlocks"));
    dfs.delete(corruptFile,true);
    LOG.info("Waiting for missing blocks count to be zero...");
    while (dfs.getMissingBlocksCount() > 0) {
      Thread.sleep(100);
    }
    assertEquals(2,dfs.getUnderReplicatedBlocksCount());
    assertEquals(2,bm.getUnderReplicatedNotMissingBlocks());
    Assert.assertEquals(0,(long)(Long)mbs.getAttribute(mxbeanName,"NumberOfMissingBlocks"));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestPersistBlocks

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

@Test public void testRestartDfsWithAbandonedBlock() throws Exception {
  final Configuration conf=new HdfsConfiguration();
  conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,0);
  MiniDFSCluster cluster=null;
  long len=0;
  FSDataOutputStream stream;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    FileSystem fs=cluster.getFileSystem();
    stream=fs.create(FILE_PATH,true,BLOCK_SIZE,(short)1,BLOCK_SIZE);
    stream.write(DATA_BEFORE_RESTART);
    stream.hflush();
    while (len < BLOCK_SIZE * (NUM_BLOCKS - 1)) {
      FileStatus status=fs.getFileStatus(FILE_PATH);
      len=status.getLen();
      Thread.sleep(100);
    }
    DFSClient dfsclient=DFSClientAdapter.getDFSClient((DistributedFileSystem)fs);
    HdfsFileStatus fileStatus=dfsclient.getNamenode().getFileInfo(FILE_NAME);
    LocatedBlocks blocks=dfsclient.getNamenode().getBlockLocations(FILE_NAME,0,BLOCK_SIZE * NUM_BLOCKS);
    assertEquals(NUM_BLOCKS,blocks.getLocatedBlocks().size());
    LocatedBlock b=blocks.getLastLocatedBlock();
    dfsclient.getNamenode().abandonBlock(b.getBlock(),fileStatus.getFileId(),FILE_NAME,dfsclient.clientName);
    cluster.restartNameNode();
    FileStatus status=fs.getFileStatus(FILE_PATH);
    assertTrue("Length incorrect: " + status.getLen(),status.getLen() == len - BLOCK_SIZE);
    FSDataInputStream readStream=fs.open(FILE_PATH);
    try {
      byte[] verifyBuf=new byte[DATA_BEFORE_RESTART.length - BLOCK_SIZE];
      IOUtils.readFully(readStream,verifyBuf,0,verifyBuf.length);
      byte[] expectedBuf=new byte[DATA_BEFORE_RESTART.length - BLOCK_SIZE];
      System.arraycopy(DATA_BEFORE_RESTART,0,expectedBuf,0,expectedBuf.length);
      assertArrayEquals(expectedBuf,verifyBuf);
    }
  finally {
      IOUtils.closeStream(readStream);
    }
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestPipelines

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Creates and closes a file of certain length.
 * Calls append to allow next write() operation to add to the end of it
 * After write() invocation, calls hflush() to make sure that data sunk through
 * the pipeline and check the state of the last block's replica.
 * It supposes to be in RBW state
 * @throws IOException in case of an error
 */
@Test public void pipeline_01() throws IOException {
  final String METHOD_NAME=GenericTestUtils.getMethodName();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Running " + METHOD_NAME);
  }
  Path filePath=new Path("/" + METHOD_NAME + ".dat");
  DFSTestUtil.createFile(fs,filePath,FILE_SIZE,REPL_FACTOR,rand.nextLong());
  if (LOG.isDebugEnabled()) {
    LOG.debug("Invoking append but doing nothing otherwise...");
  }
  FSDataOutputStream ofs=fs.append(filePath);
  ofs.writeBytes("Some more stuff to write");
  ((DFSOutputStream)ofs.getWrappedStream()).hflush();
  List lb=cluster.getNameNodeRpc().getBlockLocations(filePath.toString(),FILE_SIZE - 1,FILE_SIZE).getLocatedBlocks();
  String bpid=cluster.getNamesystem().getBlockPoolId();
  for (  DataNode dn : cluster.getDataNodes()) {
    Replica r=DataNodeTestUtils.fetchReplicaInfo(dn,bpid,lb.get(0).getBlock().getBlockId());
    assertTrue("Replica on DN " + dn + " shouldn't be null",r != null);
    assertEquals("Should be RBW replica on " + dn + " after sequence of calls append()/write()/hflush()",HdfsServerConstants.ReplicaState.RBW,r.getState());
  }
  ofs.close();
}

Class: org.apache.hadoop.hdfs.TestReplaceDatanodeOnFailure

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test replace datanode on failure. 
 */
@Test public void testReplaceDatanodeOnFailure() throws Exception {
  final Configuration conf=new HdfsConfiguration();
  ReplaceDatanodeOnFailure.ALWAYS.write(conf);
  final String[] racks=new String[REPLICATION];
  Arrays.fill(racks,RACK0);
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).racks(racks).numDataNodes(REPLICATION).build();
  try {
    final DistributedFileSystem fs=cluster.getFileSystem();
    final Path dir=new Path(DIR);
    final SlowWriter[] slowwriters=new SlowWriter[10];
    for (int i=1; i <= slowwriters.length; i++) {
      slowwriters[i - 1]=new SlowWriter(fs,new Path(dir,"file" + i),i * 200L);
    }
    for (    SlowWriter s : slowwriters) {
      s.start();
    }
    sleepSeconds(1);
    cluster.startDataNodes(conf,2,true,null,new String[]{RACK1,RACK1});
    cluster.stopDataNode(AppendTestUtil.nextInt(REPLICATION));
    sleepSeconds(5);
    for (    SlowWriter s : slowwriters) {
      s.checkReplication();
      s.interruptRunning();
    }
    for (    SlowWriter s : slowwriters) {
      s.joinAndClose();
    }
    LOG.info("Verify the file");
    for (int i=0; i < slowwriters.length; i++) {
      LOG.info(slowwriters[i].filepath + ": length=" + fs.getFileStatus(slowwriters[i].filepath).getLen());
      FSDataInputStream in=null;
      try {
        in=fs.open(slowwriters[i].filepath);
        for (int j=0, x; (x=in.read()) != -1; j++) {
          Assert.assertEquals(j,x);
        }
      }
  finally {
        IOUtils.closeStream(in);
      }
    }
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestReplication

APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testPendingReplicationRetry() throws IOException {
  MiniDFSCluster cluster=null;
  int numDataNodes=4;
  String testFile="/replication-test-file";
  Path testPath=new Path(testFile);
  byte buffer[]=new byte[1024];
  for (int i=0; i < buffer.length; i++) {
    buffer[i]='1';
  }
  try {
    Configuration conf=new HdfsConfiguration();
    conf.set(DFSConfigKeys.DFS_REPLICATION_KEY,Integer.toString(numDataNodes));
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
    cluster.waitActive();
    DFSClient dfsClient=new DFSClient(new InetSocketAddress("localhost",cluster.getNameNodePort()),conf);
    OutputStream out=cluster.getFileSystem().create(testPath);
    out.write(buffer);
    out.close();
    waitForBlockReplication(testFile,dfsClient.getNamenode(),numDataNodes,-1);
    ExtendedBlock block=dfsClient.getNamenode().getBlockLocations(testFile,0,Long.MAX_VALUE).get(0).getBlock();
    cluster.shutdown();
    cluster=null;
    for (int i=0; i < 25; i++) {
      buffer[i]='0';
    }
    int fileCount=0;
    for (int dnIndex=0; dnIndex < 3; dnIndex++) {
      File blockFile=MiniDFSCluster.getBlockFile(dnIndex,block);
      LOG.info("Checking for file " + blockFile);
      if (blockFile != null && blockFile.exists()) {
        if (fileCount == 0) {
          LOG.info("Deleting file " + blockFile);
          assertTrue(blockFile.delete());
        }
 else {
          LOG.info("Corrupting file " + blockFile);
          long len=blockFile.length();
          assertTrue(len > 50);
          RandomAccessFile blockOut=new RandomAccessFile(blockFile,"rw");
          try {
            blockOut.seek(len / 3);
            blockOut.write(buffer,0,25);
          }
  finally {
            blockOut.close();
          }
        }
        fileCount++;
      }
    }
    assertEquals(3,fileCount);
    LOG.info("Restarting minicluster after deleting a replica and corrupting 2 crcs");
    conf=new HdfsConfiguration();
    conf.set(DFSConfigKeys.DFS_REPLICATION_KEY,Integer.toString(numDataNodes));
    conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY,Integer.toString(2));
    conf.set("dfs.datanode.block.write.timeout.sec",Integer.toString(5));
    conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY,"0.75f");
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes * 2).format(false).build();
    cluster.waitActive();
    dfsClient=new DFSClient(new InetSocketAddress("localhost",cluster.getNameNodePort()),conf);
    waitForBlockReplication(testFile,dfsClient.getNamenode(),numDataNodes,-1);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testBadBlockReportOnTransfer() throws Exception {
  Configuration conf=new HdfsConfiguration();
  FileSystem fs=null;
  DFSClient dfsClient=null;
  LocatedBlocks blocks=null;
  int replicaCount=0;
  short replFactor=1;
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  cluster.waitActive();
  fs=cluster.getFileSystem();
  dfsClient=new DFSClient(new InetSocketAddress("localhost",cluster.getNameNodePort()),conf);
  Path file1=new Path("/tmp/testBadBlockReportOnTransfer/file1");
  DFSTestUtil.createFile(fs,file1,1024,replFactor,0);
  DFSTestUtil.waitReplication(fs,file1,replFactor);
  ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,file1);
  int blockFilesCorrupted=cluster.corruptBlockOnDataNodes(block);
  assertEquals("Corrupted too few blocks",replFactor,blockFilesCorrupted);
  replFactor=2;
  fs.setReplication(file1,replFactor);
  blocks=dfsClient.getNamenode().getBlockLocations(file1.toString(),0,Long.MAX_VALUE);
  while (blocks.get(0).isCorrupt() != true) {
    try {
      LOG.info("Waiting until block is marked as corrupt...");
      Thread.sleep(1000);
    }
 catch (    InterruptedException ie) {
    }
    blocks=dfsClient.getNamenode().getBlockLocations(file1.toString(),0,Long.MAX_VALUE);
  }
  replicaCount=blocks.get(0).getLocations().length;
  assertTrue(replicaCount == 1);
  cluster.shutdown();
}

Class: org.apache.hadoop.hdfs.TestRollingUpgrade

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testRollingUpgradeWithQJM() throws Exception {
  String nnDirPrefix=MiniDFSCluster.getBaseDirectory() + "/nn/";
  final File nn1Dir=new File(nnDirPrefix + "image1");
  final File nn2Dir=new File(nnDirPrefix + "image2");
  LOG.info("nn1Dir=" + nn1Dir);
  LOG.info("nn2Dir=" + nn2Dir);
  final Configuration conf=new HdfsConfiguration();
  final MiniJournalCluster mjc=new MiniJournalCluster.Builder(conf).build();
  setConf(conf,nn1Dir,mjc);
{
    final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).manageNameDfsDirs(false).checkExitOnShutdown(false).build();
    cluster.shutdown();
  }
  MiniDFSCluster cluster2=null;
  try {
    FileUtil.fullyDelete(nn2Dir);
    FileUtil.copy(nn1Dir,FileSystem.getLocal(conf).getRaw(),new Path(nn2Dir.getAbsolutePath()),false,conf);
    final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).manageNameDfsDirs(false).checkExitOnShutdown(false).build();
    final Path foo=new Path("/foo");
    final Path bar=new Path("/bar");
    final Path baz=new Path("/baz");
    final RollingUpgradeInfo info1;
{
      final DistributedFileSystem dfs=cluster.getFileSystem();
      dfs.mkdirs(foo);
      dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
      info1=dfs.rollingUpgrade(RollingUpgradeAction.PREPARE);
      dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
      LOG.info("START\n" + info1);
      Assert.assertEquals(info1,dfs.rollingUpgrade(RollingUpgradeAction.QUERY));
      dfs.mkdirs(bar);
      cluster.shutdown();
    }
    final Configuration conf2=setConf(new Configuration(),nn2Dir,mjc);
    cluster2=new MiniDFSCluster.Builder(conf2).numDataNodes(0).format(false).manageNameDfsDirs(false).build();
    final DistributedFileSystem dfs2=cluster2.getFileSystem();
    Assert.assertTrue(dfs2.exists(foo));
    Assert.assertTrue(dfs2.exists(bar));
    Assert.assertFalse(dfs2.exists(baz));
    Assert.assertEquals(info1,dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
    dfs2.mkdirs(baz);
    LOG.info("RESTART cluster 2");
    cluster2.restartNameNode();
    Assert.assertEquals(info1,dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
    Assert.assertTrue(dfs2.exists(foo));
    Assert.assertTrue(dfs2.exists(bar));
    Assert.assertTrue(dfs2.exists(baz));
    try {
      cluster2.restartNameNode("-upgrade");
    }
 catch (    IOException e) {
      LOG.info("The exception is expected.",e);
    }
    LOG.info("RESTART cluster 2 again");
    cluster2.restartNameNode();
    Assert.assertEquals(info1,dfs2.rollingUpgrade(RollingUpgradeAction.QUERY));
    Assert.assertTrue(dfs2.exists(foo));
    Assert.assertTrue(dfs2.exists(bar));
    Assert.assertTrue(dfs2.exists(baz));
    final RollingUpgradeInfo finalize=dfs2.rollingUpgrade(RollingUpgradeAction.FINALIZE);
    LOG.info("FINALIZE: " + finalize);
    Assert.assertEquals(info1.getStartTime(),finalize.getStartTime());
    LOG.info("RESTART cluster 2 with regular startup option");
    cluster2.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
    cluster2.restartNameNode();
    Assert.assertTrue(dfs2.exists(foo));
    Assert.assertTrue(dfs2.exists(bar));
    Assert.assertTrue(dfs2.exists(baz));
  }
  finally {
    if (cluster2 != null)     cluster2.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.TestRollingUpgradeRollback

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testRollbackCommand() throws Exception {
  final Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  final Path foo=new Path("/foo");
  final Path bar=new Path("/bar");
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    final DistributedFileSystem dfs=cluster.getFileSystem();
    final DFSAdmin dfsadmin=new DFSAdmin(conf);
    dfs.mkdirs(foo);
    dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    Assert.assertEquals(0,dfsadmin.run(new String[]{"-rollingUpgrade","prepare"}));
    dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    dfs.mkdirs(bar);
    NNStorage storage=cluster.getNamesystem().getFSImage().getStorage();
    checkNNStorage(storage,3,-1);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
  NameNode nn=null;
  try {
    nn=NameNode.createNameNode(new String[]{"-rollingUpgrade","rollback"},conf);
    INode fooNode=nn.getNamesystem().getFSDirectory().getINode4Write(foo.toString());
    Assert.assertNotNull(fooNode);
    INode barNode=nn.getNamesystem().getFSDirectory().getINode4Write(bar.toString());
    Assert.assertNull(barNode);
    NNStorage storage=nn.getNamesystem().getFSImage().getStorage();
    checkNNStorage(storage,3,7);
  }
  finally {
    if (nn != null) {
      nn.stop();
      nn.join();
    }
  }
}

Class: org.apache.hadoop.hdfs.TestSafeMode

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that the NN initializes its under-replicated blocks queue
 * before it is ready to exit safemode (HDFS-1476)
 */
@Test(timeout=45000) public void testInitializeReplQueuesEarly() throws Exception {
  LOG.info("Starting testInitializeReplQueuesEarly");
  BlockManagerTestUtil.setWritingPrefersLocalNode(cluster.getNamesystem().getBlockManager(),false);
  cluster.startDataNodes(conf,2,true,StartupOption.REGULAR,null);
  cluster.waitActive();
  LOG.info("Creating files");
  DFSTestUtil.createFile(fs,TEST_PATH,15 * BLOCK_SIZE,(short)1,1L);
  LOG.info("Stopping all DataNodes");
  List dnprops=Lists.newLinkedList();
  dnprops.add(cluster.stopDataNode(0));
  dnprops.add(cluster.stopDataNode(0));
  dnprops.add(cluster.stopDataNode(0));
  cluster.getConfiguration(0).setFloat(DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY,1f / 15f);
  LOG.info("Restarting NameNode");
  cluster.restartNameNode();
  final NameNode nn=cluster.getNameNode();
  String status=nn.getNamesystem().getSafemode();
  assertEquals("Safe mode is ON. The reported blocks 0 needs additional " + "15 blocks to reach the threshold 0.9990 of total blocks 15.\n" + "The number of live datanodes 0 has reached the minimum number 0. "+ "Safe mode will be turned off automatically once the thresholds "+ "have been reached.",status);
  assertFalse("Mis-replicated block queues should not be initialized " + "until threshold is crossed",NameNodeAdapter.safeModeInitializedReplQueues(nn));
  LOG.info("Restarting one DataNode");
  cluster.restartDataNode(dnprops.remove(0));
  GenericTestUtils.waitFor(new Supplier(){
    @Override public Boolean get(){
      return getLongCounter("StorageBlockReportOps",getMetrics(NN_METRICS)) == cluster.getStoragesPerDatanode();
    }
  }
,10,10000);
  final int safe=NameNodeAdapter.getSafeModeSafeBlocks(nn);
  assertTrue("Expected first block report to make some blocks safe.",safe > 0);
  assertTrue("Did not expect first block report to make all blocks safe.",safe < 15);
  assertTrue(NameNodeAdapter.safeModeInitializedReplQueues(nn));
  BlockManagerTestUtil.updateState(nn.getNamesystem().getBlockManager());
  long underReplicatedBlocks=nn.getNamesystem().getUnderReplicatedBlocks();
  while (underReplicatedBlocks != (15 - safe)) {
    LOG.info("UnderReplicatedBlocks expected=" + (15 - safe) + ", actual="+ underReplicatedBlocks);
    Thread.sleep(100);
    BlockManagerTestUtil.updateState(nn.getNamesystem().getBlockManager());
    underReplicatedBlocks=nn.getNamesystem().getUnderReplicatedBlocks();
  }
  cluster.restartDataNodes();
}

Class: org.apache.hadoop.hdfs.TestWriteRead

APIUtilityVerifier EqualityVerifier 

/** 
 * Junit Test reading while writing. 
 */
@Test public void testWriteReadSeq() throws IOException {
  useFCOption=false;
  positionReadOption=false;
  String fname=filenameOption;
  long rdBeginPos=0;
  int stat=testWriteAndRead(fname,WR_NTIMES,WR_CHUNK_SIZE,rdBeginPos);
  LOG.info("Summary status from test1: status= " + stat);
  Assert.assertEquals(0,stat);
}

APIUtilityVerifier EqualityVerifier 

/** 
 * Junit Test position read while writing. 
 */
@Test public void testWriteReadPos() throws IOException {
  String fname=filenameOption;
  positionReadOption=true;
  long rdBeginPos=0;
  int stat=testWriteAndRead(fname,WR_NTIMES,WR_CHUNK_SIZE,rdBeginPos);
  Assert.assertEquals(0,stat);
}

APIUtilityVerifier EqualityVerifier 

/** 
 * Junit Test position read of the current block being written. 
 */
@Test public void testReadPosCurrentBlock() throws IOException {
  String fname=filenameOption;
  positionReadOption=true;
  int wrChunkSize=(int)(blockSize) + (int)(blockSize / 2);
  long rdBeginPos=blockSize + 1;
  int numTimes=5;
  int stat=testWriteAndRead(fname,numTimes,wrChunkSize,rdBeginPos);
  Assert.assertEquals(0,stat);
}

Class: org.apache.hadoop.hdfs.nfs.nfs3.TestDFSClientCache

APIUtilityVerifier InternalCallVerifier ConditionMatcher 

@Test public void testGetUserGroupInformationSecure() throws IOException {
  String userName="user1";
  String currentUser="test-user";
  NfsConfiguration conf=new NfsConfiguration();
  UserGroupInformation currentUserUgi=UserGroupInformation.createRemoteUser(currentUser);
  currentUserUgi.setAuthenticationMethod(KERBEROS);
  UserGroupInformation.setLoginUser(currentUserUgi);
  DFSClientCache cache=new DFSClientCache(conf);
  UserGroupInformation ugiResult=cache.getUserGroupInformation(userName,currentUserUgi);
  assertThat(ugiResult.getUserName(),is(userName));
  assertThat(ugiResult.getRealUser(),is(currentUserUgi));
  assertThat(ugiResult.getAuthenticationMethod(),is(UserGroupInformation.AuthenticationMethod.PROXY));
}

APIUtilityVerifier InternalCallVerifier ConditionMatcher 

@Test public void testGetUserGroupInformation() throws IOException {
  String userName="user1";
  String currentUser="currentUser";
  UserGroupInformation currentUserUgi=UserGroupInformation.createUserForTesting(currentUser,new String[0]);
  NfsConfiguration conf=new NfsConfiguration();
  conf.set(FileSystem.FS_DEFAULT_NAME_KEY,"hdfs://localhost");
  DFSClientCache cache=new DFSClientCache(conf);
  UserGroupInformation ugiResult=cache.getUserGroupInformation(userName,currentUserUgi);
  assertThat(ugiResult.getUserName(),is(userName));
  assertThat(ugiResult.getRealUser(),is(currentUserUgi));
  assertThat(ugiResult.getAuthenticationMethod(),is(UserGroupInformation.AuthenticationMethod.PROXY));
}

Class: org.apache.hadoop.hdfs.nfs.nfs3.TestReaddir

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testReaddirPlus() throws IOException {
  HdfsFileStatus status=nn.getRpcServer().getFileInfo(testdir);
  long dirId=status.getFileId();
  XDR xdr_req=new XDR();
  FileHandle handle=new FileHandle(dirId);
  handle.serialize(xdr_req);
  xdr_req.writeLongAsHyper(0);
  xdr_req.writeLongAsHyper(0);
  xdr_req.writeInt(100);
  xdr_req.writeInt(1000);
  READDIRPLUS3Response responsePlus=nfsd.readdirplus(xdr_req.asReadOnlyWrap(),securityHandler,new InetSocketAddress("localhost",1234));
  List direntPlus=responsePlus.getDirListPlus().getEntries();
  assertTrue(direntPlus.size() == 5);
  status=nn.getRpcServer().getFileInfo(testdir + "/f2");
  long f2Id=status.getFileId();
  xdr_req=new XDR();
  handle=new FileHandle(dirId);
  handle.serialize(xdr_req);
  xdr_req.writeLongAsHyper(f2Id);
  xdr_req.writeLongAsHyper(0);
  xdr_req.writeInt(100);
  xdr_req.writeInt(1000);
  responsePlus=nfsd.readdirplus(xdr_req.asReadOnlyWrap(),securityHandler,new InetSocketAddress("localhost",1234));
  direntPlus=responsePlus.getDirListPlus().getEntries();
  assertTrue(direntPlus.size() == 1);
  EntryPlus3 entryPlus=direntPlus.get(0);
  assertTrue(entryPlus.getName().equals("f3"));
  hdfs.delete(new Path(testdir + "/f2"),false);
  responsePlus=nfsd.readdirplus(xdr_req.asReadOnlyWrap(),securityHandler,new InetSocketAddress("localhost",1234));
  direntPlus=responsePlus.getDirListPlus().getEntries();
  assertTrue(direntPlus.size() == 2);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testReaddirBasic() throws IOException {
  HdfsFileStatus status=nn.getRpcServer().getFileInfo(testdir);
  long dirId=status.getFileId();
  XDR xdr_req=new XDR();
  FileHandle handle=new FileHandle(dirId);
  handle.serialize(xdr_req);
  xdr_req.writeLongAsHyper(0);
  xdr_req.writeLongAsHyper(0);
  xdr_req.writeInt(100);
  READDIR3Response response=nfsd.readdir(xdr_req.asReadOnlyWrap(),securityHandler,new InetSocketAddress("localhost",1234));
  List dirents=response.getDirList().getEntries();
  assertTrue(dirents.size() == 5);
  status=nn.getRpcServer().getFileInfo(testdir + "/f2");
  long f2Id=status.getFileId();
  xdr_req=new XDR();
  handle=new FileHandle(dirId);
  handle.serialize(xdr_req);
  xdr_req.writeLongAsHyper(f2Id);
  xdr_req.writeLongAsHyper(0);
  xdr_req.writeInt(100);
  response=nfsd.readdir(xdr_req.asReadOnlyWrap(),securityHandler,new InetSocketAddress("localhost",1234));
  dirents=response.getDirList().getEntries();
  assertTrue(dirents.size() == 1);
  Entry3 entry=dirents.get(0);
  assertTrue(entry.getName().equals("f3"));
  hdfs.delete(new Path(testdir + "/f2"),false);
  response=nfsd.readdir(xdr_req.asReadOnlyWrap(),securityHandler,new InetSocketAddress("localhost",1234));
  dirents=response.getDirList().getEntries();
  assertTrue(dirents.size() == 2);
}

Class: org.apache.hadoop.hdfs.nfs.nfs3.TestWrites

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testCheckCommit() throws IOException {
  DFSClient dfsClient=Mockito.mock(DFSClient.class);
  Nfs3FileAttributes attr=new Nfs3FileAttributes();
  HdfsDataOutputStream fos=Mockito.mock(HdfsDataOutputStream.class);
  Mockito.when(fos.getPos()).thenReturn((long)0);
  OpenFileCtx ctx=new OpenFileCtx(fos,attr,"/dumpFilePath",dfsClient,new IdUserGroup(new NfsConfiguration()));
  COMMIT_STATUS ret;
  ctx.setActiveStatusForTest(false);
  Channel ch=Mockito.mock(Channel.class);
  ret=ctx.checkCommit(dfsClient,0,ch,1,attr,false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_INACTIVE_CTX);
  ctx.getPendingWritesForTest().put(new OffsetRange(5,10),new WriteCtx(null,0,0,0,null,null,null,0,false,null));
  ret=ctx.checkCommit(dfsClient,0,ch,1,attr,false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_INACTIVE_WITH_PENDING_WRITE);
  ctx.setActiveStatusForTest(true);
  Mockito.when(fos.getPos()).thenReturn((long)10);
  COMMIT_STATUS status=ctx.checkCommitInternal(5,null,1,attr,false);
  Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  ret=ctx.checkCommit(dfsClient,5,ch,1,attr,false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
  status=ctx.checkCommitInternal(10,ch,1,attr,false);
  Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
  ret=ctx.checkCommit(dfsClient,10,ch,1,attr,false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
  ConcurrentNavigableMap commits=ctx.getPendingCommitsForTest();
  Assert.assertTrue(commits.size() == 0);
  ret=ctx.checkCommit(dfsClient,11,ch,1,attr,false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_WAIT);
  Assert.assertTrue(commits.size() == 1);
  long key=commits.firstKey();
  Assert.assertTrue(key == 11);
  commits.remove(new Long(11));
  ret=ctx.checkCommit(dfsClient,0,ch,1,attr,false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_WAIT);
  Assert.assertTrue(commits.size() == 1);
  key=commits.firstKey();
  Assert.assertTrue(key == 9);
  ctx.getPendingWritesForTest().remove(new OffsetRange(5,10));
  ret=ctx.checkCommit(dfsClient,0,ch,1,attr,false);
  Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier PublicFieldVerifier 

@Test public void testAlterWriteRequest() throws IOException {
  int len=20;
  byte[] data=new byte[len];
  ByteBuffer buffer=ByteBuffer.wrap(data);
  for (int i=0; i < len; i++) {
    buffer.put((byte)i);
  }
  buffer.flip();
  int originalCount=buffer.array().length;
  WRITE3Request request=new WRITE3Request(new FileHandle(),0,data.length,WriteStableHow.UNSTABLE,buffer);
  WriteCtx writeCtx1=new WriteCtx(request.getHandle(),request.getOffset(),request.getCount(),WriteCtx.INVALID_ORIGINAL_COUNT,request.getStableHow(),request.getData(),null,1,false,WriteCtx.DataState.NO_DUMP);
  Assert.assertTrue(writeCtx1.getData().array().length == originalCount);
  OpenFileCtx.alterWriteRequest(request,12);
  WriteCtx writeCtx2=new WriteCtx(request.getHandle(),request.getOffset(),request.getCount(),originalCount,request.getStableHow(),request.getData(),null,2,false,WriteCtx.DataState.NO_DUMP);
  ByteBuffer appendedData=writeCtx2.getData();
  int position=appendedData.position();
  int limit=appendedData.limit();
  Assert.assertTrue(position == 12);
  Assert.assertTrue(limit - position == 8);
  Assert.assertTrue(appendedData.get(position) == (byte)12);
  Assert.assertTrue(appendedData.get(position + 1) == (byte)13);
  Assert.assertTrue(appendedData.get(position + 2) == (byte)14);
  Assert.assertTrue(appendedData.get(position + 7) == (byte)19);
  buffer.position(0);
  request=new WRITE3Request(new FileHandle(),0,data.length,WriteStableHow.UNSTABLE,buffer);
  OpenFileCtx.alterWriteRequest(request,1);
  WriteCtx writeCtx3=new WriteCtx(request.getHandle(),request.getOffset(),request.getCount(),originalCount,request.getStableHow(),request.getData(),null,2,false,WriteCtx.DataState.NO_DUMP);
  appendedData=writeCtx3.getData();
  position=appendedData.position();
  limit=appendedData.limit();
  Assert.assertTrue(position == 1);
  Assert.assertTrue(limit - position == 19);
  Assert.assertTrue(appendedData.get(position) == (byte)1);
  Assert.assertTrue(appendedData.get(position + 18) == (byte)19);
  buffer.position(0);
  request=new WRITE3Request(new FileHandle(),0,data.length,WriteStableHow.UNSTABLE,buffer);
  OpenFileCtx.alterWriteRequest(request,19);
  WriteCtx writeCtx4=new WriteCtx(request.getHandle(),request.getOffset(),request.getCount(),originalCount,request.getStableHow(),request.getData(),null,2,false,WriteCtx.DataState.NO_DUMP);
  appendedData=writeCtx4.getData();
  position=appendedData.position();
  limit=appendedData.limit();
  Assert.assertTrue(position == 19);
  Assert.assertTrue(limit - position == 1);
  Assert.assertTrue(appendedData.get(position) == (byte)19);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testWriteStableHow() throws IOException, InterruptedException {
  NfsConfiguration config=new NfsConfiguration();
  DFSClient client=null;
  MiniDFSCluster cluster=null;
  RpcProgramNfs3 nfsd;
  SecurityHandler securityHandler=Mockito.mock(SecurityHandler.class);
  Mockito.when(securityHandler.getUser()).thenReturn(System.getProperty("user.name"));
  String currentUser=System.getProperty("user.name");
  config.set(DefaultImpersonationProvider.getTestProvider().getProxySuperuserGroupConfKey(currentUser),"*");
  config.set(DefaultImpersonationProvider.getTestProvider().getProxySuperuserIpConfKey(currentUser),"*");
  ProxyUsers.refreshSuperUserGroupsConfiguration(config);
  try {
    cluster=new MiniDFSCluster.Builder(config).numDataNodes(1).build();
    cluster.waitActive();
    client=new DFSClient(NameNode.getAddress(config),config);
    config.setInt("nfs3.mountd.port",0);
    config.setInt("nfs3.server.port",0);
    Nfs3 nfs3=new Nfs3(config);
    nfs3.startServiceInternal(false);
    nfsd=(RpcProgramNfs3)nfs3.getRpcProgram();
    HdfsFileStatus status=client.getFileInfo("/");
    FileHandle rootHandle=new FileHandle(status.getFileId());
    CREATE3Request createReq=new CREATE3Request(rootHandle,"file1",Nfs3Constant.CREATE_UNCHECKED,new SetAttr3(),0);
    XDR createXdr=new XDR();
    createReq.serialize(createXdr);
    CREATE3Response createRsp=nfsd.create(createXdr.asReadOnlyWrap(),securityHandler,new InetSocketAddress("localhost",1234));
    FileHandle handle=createRsp.getObjHandle();
    byte[] buffer=new byte[10];
    for (int i=0; i < 10; i++) {
      buffer[i]=(byte)i;
    }
    WRITE3Request writeReq=new WRITE3Request(handle,0,10,WriteStableHow.DATA_SYNC,ByteBuffer.wrap(buffer));
    XDR writeXdr=new XDR();
    writeReq.serialize(writeXdr);
    nfsd.write(writeXdr.asReadOnlyWrap(),null,1,securityHandler,new InetSocketAddress("localhost",1234));
    waitWrite(nfsd,handle,60000);
    READ3Request readReq=new READ3Request(handle,0,10);
    XDR readXdr=new XDR();
    readReq.serialize(readXdr);
    READ3Response readRsp=nfsd.read(readXdr.asReadOnlyWrap(),securityHandler,new InetSocketAddress("localhost",1234));
    assertTrue(Arrays.equals(buffer,readRsp.getData().array()));
    CREATE3Request createReq2=new CREATE3Request(rootHandle,"file2",Nfs3Constant.CREATE_UNCHECKED,new SetAttr3(),0);
    XDR createXdr2=new XDR();
    createReq2.serialize(createXdr2);
    CREATE3Response createRsp2=nfsd.create(createXdr2.asReadOnlyWrap(),securityHandler,new InetSocketAddress("localhost",1234));
    FileHandle handle2=createRsp2.getObjHandle();
    WRITE3Request writeReq2=new WRITE3Request(handle2,0,10,WriteStableHow.FILE_SYNC,ByteBuffer.wrap(buffer));
    XDR writeXdr2=new XDR();
    writeReq2.serialize(writeXdr2);
    nfsd.write(writeXdr2.asReadOnlyWrap(),null,1,securityHandler,new InetSocketAddress("localhost",1234));
    waitWrite(nfsd,handle2,60000);
    READ3Request readReq2=new READ3Request(handle2,0,10);
    XDR readXdr2=new XDR();
    readReq2.serialize(readXdr2);
    READ3Response readRsp2=nfsd.read(readXdr2.asReadOnlyWrap(),securityHandler,new InetSocketAddress("localhost",1234));
    assertTrue(Arrays.equals(buffer,readRsp2.getData().array()));
    status=client.getFileInfo("/file2");
    assertTrue(status.getLen() == 10);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.protocolPB.TestPBHelper

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertCheckpointSignature(){
  CheckpointSignature s=new CheckpointSignature(getStorageInfo(NodeType.NAME_NODE),"bpid",100,1);
  CheckpointSignatureProto sProto=PBHelper.convert(s);
  CheckpointSignature s1=PBHelper.convert(sProto);
  assertEquals(s.getBlockpoolID(),s1.getBlockpoolID());
  assertEquals(s.getClusterID(),s1.getClusterID());
  assertEquals(s.getCTime(),s1.getCTime());
  assertEquals(s.getCurSegmentTxId(),s1.getCurSegmentTxId());
  assertEquals(s.getLayoutVersion(),s1.getLayoutVersion());
  assertEquals(s.getMostRecentCheckpointTxId(),s1.getMostRecentCheckpointTxId());
  assertEquals(s.getNamespaceID(),s1.getNamespaceID());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertRecoveringBlock(){
  DatanodeInfo di1=DFSTestUtil.getLocalDatanodeInfo();
  DatanodeInfo di2=DFSTestUtil.getLocalDatanodeInfo();
  DatanodeInfo[] dnInfo=new DatanodeInfo[]{di1,di2};
  RecoveringBlock b=new RecoveringBlock(getExtendedBlock(),dnInfo,3);
  RecoveringBlockProto bProto=PBHelper.convert(b);
  RecoveringBlock b1=PBHelper.convert(bProto);
  assertEquals(b.getBlock(),b1.getBlock());
  DatanodeInfo[] dnInfo1=b1.getLocations();
  assertEquals(dnInfo.length,dnInfo1.length);
  for (int i=0; i < dnInfo.length; i++) {
    compare(dnInfo[0],dnInfo1[0]);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertRemoteEditLogManifest(){
  List logs=new ArrayList();
  logs.add(new RemoteEditLog(1,10));
  logs.add(new RemoteEditLog(11,20));
  RemoteEditLogManifest m=new RemoteEditLogManifest(logs);
  RemoteEditLogManifestProto mProto=PBHelper.convert(m);
  RemoteEditLogManifest m1=PBHelper.convert(mProto);
  List logs1=m1.getLogs();
  assertEquals(logs.size(),logs1.size());
  for (int i=0; i < logs.size(); i++) {
    compare(logs.get(i),logs1.get(i));
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertNamenodeRegistration(){
  StorageInfo info=getStorageInfo(NodeType.NAME_NODE);
  NamenodeRegistration reg=new NamenodeRegistration("address:999","http:1000",info,NamenodeRole.NAMENODE);
  NamenodeRegistrationProto regProto=PBHelper.convert(reg);
  NamenodeRegistration reg2=PBHelper.convert(regProto);
  assertEquals(reg.getAddress(),reg2.getAddress());
  assertEquals(reg.getClusterID(),reg2.getClusterID());
  assertEquals(reg.getCTime(),reg2.getCTime());
  assertEquals(reg.getHttpAddress(),reg2.getHttpAddress());
  assertEquals(reg.getLayoutVersion(),reg2.getLayoutVersion());
  assertEquals(reg.getNamespaceID(),reg2.getNamespaceID());
  assertEquals(reg.getRegistrationID(),reg2.getRegistrationID());
  assertEquals(reg.getRole(),reg2.getRole());
  assertEquals(reg.getVersion(),reg2.getVersion());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertExtendedBlock(){
  ExtendedBlock b=getExtendedBlock();
  ExtendedBlockProto bProto=PBHelper.convert(b);
  ExtendedBlock b1=PBHelper.convert(bProto);
  assertEquals(b,b1);
  b.setBlockId(-1);
  bProto=PBHelper.convert(b);
  b1=PBHelper.convert(bProto);
  assertEquals(b,b1);
}

APIUtilityVerifier EqualityVerifier 

@Test public void testConvertLocatedBlockList(){
  ArrayList lbl=new ArrayList();
  for (int i=0; i < 3; i++) {
    lbl.add(createLocatedBlock());
  }
  List lbpl=PBHelper.convertLocatedBlock2(lbl);
  List lbl2=PBHelper.convertLocatedBlock(lbpl);
  assertEquals(lbl.size(),lbl2.size());
  for (int i=0; i < lbl.size(); i++) {
    compare(lbl.get(i),lbl2.get(2));
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertStoragInfo(){
  StorageInfo info=getStorageInfo(NodeType.NAME_NODE);
  StorageInfoProto infoProto=PBHelper.convert(info);
  StorageInfo info2=PBHelper.convert(infoProto,NodeType.NAME_NODE);
  assertEquals(info.getClusterID(),info2.getClusterID());
  assertEquals(info.getCTime(),info2.getCTime());
  assertEquals(info.getLayoutVersion(),info2.getLayoutVersion());
  assertEquals(info.getNamespaceID(),info2.getNamespaceID());
}

APIUtilityVerifier EqualityVerifier 

@Test public void testAclStatusProto(){
  AclEntry e=new AclEntry.Builder().setName("test").setPermission(FsAction.READ_EXECUTE).setScope(AclEntryScope.DEFAULT).setType(AclEntryType.OTHER).build();
  AclStatus s=new AclStatus.Builder().owner("foo").group("bar").addEntry(e).build();
  Assert.assertEquals(s,PBHelper.convert(PBHelper.convert(s)));
}

APIUtilityVerifier EqualityVerifier 

@Test public void testConvertLocatedBlockArray(){
  LocatedBlock[] lbl=new LocatedBlock[3];
  for (int i=0; i < 3; i++) {
    lbl[i]=createLocatedBlock();
  }
  LocatedBlockProto[] lbpl=PBHelper.convertLocatedBlock(lbl);
  LocatedBlock[] lbl2=PBHelper.convertLocatedBlock(lbpl);
  assertEquals(lbl.length,lbl2.length);
  for (int i=0; i < lbl.length; i++) {
    compare(lbl[i],lbl2[i]);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertNamespaceInfo(){
  NamespaceInfo info=new NamespaceInfo(37,"clusterID","bpID",2300);
  NamespaceInfoProto proto=PBHelper.convert(info);
  NamespaceInfo info2=PBHelper.convert(proto);
  compare(info,info2);
  assertEquals(info.getBlockPoolID(),info2.getBlockPoolID());
  assertEquals(info.getBuildVersion(),info2.getBuildVersion());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertDatanodeRegistration(){
  DatanodeID dnId=DFSTestUtil.getLocalDatanodeID();
  BlockKey[] keys=new BlockKey[]{getBlockKey(2),getBlockKey(3)};
  ExportedBlockKeys expKeys=new ExportedBlockKeys(true,9,10,getBlockKey(1),keys);
  DatanodeRegistration reg=new DatanodeRegistration(dnId,new StorageInfo(NodeType.DATA_NODE),expKeys,"3.0.0");
  DatanodeRegistrationProto proto=PBHelper.convert(reg);
  DatanodeRegistration reg2=PBHelper.convert(proto);
  compare(reg.getStorageInfo(),reg2.getStorageInfo());
  compare(reg.getExportedKeys(),reg2.getExportedKeys());
  compare(reg,reg2);
  assertEquals(reg.getSoftwareVersion(),reg2.getSoftwareVersion());
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertBlockCommand(){
  Block[] blocks=new Block[]{new Block(21),new Block(22)};
  DatanodeInfo[][] dnInfos=new DatanodeInfo[][]{new DatanodeInfo[1],new DatanodeInfo[2]};
  dnInfos[0][0]=DFSTestUtil.getLocalDatanodeInfo();
  dnInfos[1][0]=DFSTestUtil.getLocalDatanodeInfo();
  dnInfos[1][1]=DFSTestUtil.getLocalDatanodeInfo();
  String[][] storageIDs={{"s00"},{"s10","s11"}};
  StorageType[][] storageTypes={{StorageType.DEFAULT},{StorageType.DEFAULT,StorageType.DEFAULT}};
  BlockCommand bc=new BlockCommand(DatanodeProtocol.DNA_TRANSFER,"bp1",blocks,dnInfos,storageTypes,storageIDs);
  BlockCommandProto bcProto=PBHelper.convert(bc);
  BlockCommand bc2=PBHelper.convert(bcProto);
  assertEquals(bc.getAction(),bc2.getAction());
  assertEquals(bc.getBlocks().length,bc2.getBlocks().length);
  Block[] blocks2=bc2.getBlocks();
  for (int i=0; i < blocks.length; i++) {
    assertEquals(blocks[i],blocks2[i]);
  }
  DatanodeInfo[][] dnInfos2=bc2.getTargets();
  assertEquals(dnInfos.length,dnInfos2.length);
  for (int i=0; i < dnInfos.length; i++) {
    DatanodeInfo[] d1=dnInfos[i];
    DatanodeInfo[] d2=dnInfos2[i];
    assertEquals(d1.length,d2.length);
    for (int j=0; j < d1.length; j++) {
      compare(d1[j],d2[j]);
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testAclEntryProto(){
  AclEntry e1=new AclEntry.Builder().setName("test").setPermission(FsAction.READ_EXECUTE).setScope(AclEntryScope.DEFAULT).setType(AclEntryType.OTHER).build();
  AclEntry e2=new AclEntry.Builder().setScope(AclEntryScope.ACCESS).setType(AclEntryType.USER).setPermission(FsAction.ALL).build();
  AclEntry e3=new AclEntry.Builder().setScope(AclEntryScope.ACCESS).setType(AclEntryType.USER).setName("test").build();
  AclEntry[] expected=new AclEntry[]{e1,e2,new AclEntry.Builder().setScope(e3.getScope()).setType(e3.getType()).setName(e3.getName()).setPermission(FsAction.NONE).build()};
  AclEntry[] actual=Lists.newArrayList(PBHelper.convertAclEntry(PBHelper.convertAclEntryProto(Lists.newArrayList(e1,e2,e3)))).toArray(new AclEntry[0]);
  Assert.assertArrayEquals(expected,actual);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertBlocksWithLocations(){
  BlockWithLocations[] list=new BlockWithLocations[]{getBlockWithLocations(1),getBlockWithLocations(2)};
  BlocksWithLocations locs=new BlocksWithLocations(list);
  BlocksWithLocationsProto locsProto=PBHelper.convert(locs);
  BlocksWithLocations locs2=PBHelper.convert(locsProto);
  BlockWithLocations[] blocks=locs.getBlocks();
  BlockWithLocations[] blocks2=locs2.getBlocks();
  assertEquals(blocks.length,blocks2.length);
  for (int i=0; i < blocks.length; i++) {
    compare(blocks[i],blocks2[i]);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testConvertBlockRecoveryCommand(){
  DatanodeInfo di1=DFSTestUtil.getLocalDatanodeInfo();
  DatanodeInfo di2=DFSTestUtil.getLocalDatanodeInfo();
  DatanodeInfo[] dnInfo=new DatanodeInfo[]{di1,di2};
  List blks=ImmutableList.of(new RecoveringBlock(getExtendedBlock(1),dnInfo,3),new RecoveringBlock(getExtendedBlock(2),dnInfo,3));
  BlockRecoveryCommand cmd=new BlockRecoveryCommand(blks);
  BlockRecoveryCommandProto proto=PBHelper.convert(cmd);
  assertEquals(1,proto.getBlocks(0).getBlock().getB().getBlockId());
  assertEquals(2,proto.getBlocks(1).getBlock().getB().getBlockId());
  BlockRecoveryCommand cmd2=PBHelper.convert(proto);
  List cmd2Blks=Lists.newArrayList(cmd2.getRecoveringBlocks());
  assertEquals(blks.get(0).getBlock(),cmd2Blks.get(0).getBlock());
  assertEquals(blks.get(1).getBlock(),cmd2Blks.get(1).getBlock());
  assertEquals(Joiner.on(",").join(blks),Joiner.on(",").join(cmd2Blks));
  assertEquals(cmd.toString(),cmd2.toString());
}

APIUtilityVerifier EqualityVerifier 

@Test public void testConvertBlock(){
  Block b=new Block(1,100,3);
  BlockProto bProto=PBHelper.convert(b);
  Block b2=PBHelper.convert(bProto);
  assertEquals(b,b2);
}

Class: org.apache.hadoop.hdfs.qjournal.TestMiniJournalCluster

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testStartStop() throws IOException {
  Configuration conf=new Configuration();
  MiniJournalCluster c=new MiniJournalCluster.Builder(conf).build();
  try {
    URI uri=c.getQuorumJournalURI("myjournal");
    String[] addrs=uri.getAuthority().split(";");
    assertEquals(3,addrs.length);
    JournalNode node=c.getJournalNode(0);
    String dir=node.getConf().get(DFSConfigKeys.DFS_JOURNALNODE_EDITS_DIR_KEY);
    assertEquals(new File(MiniDFSCluster.getBaseDirectory() + "journalnode-0").getAbsolutePath(),dir);
  }
  finally {
    c.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.qjournal.client.TestQJMWithFaults

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Sets up two of the nodes to each drop a single RPC, at all
 * possible combinations of RPCs. This may result in the
 * active writer failing to write. After this point, a new writer
 * should be able to recover and continue writing without
 * data loss.
 */
@Test public void testRecoverAfterDoubleFailures() throws Exception {
  final long MAX_IPC_NUMBER=determineMaxIpcNumber();
  for (int failA=1; failA <= MAX_IPC_NUMBER; failA++) {
    for (int failB=1; failB <= MAX_IPC_NUMBER; failB++) {
      String injectionStr="(" + failA + ", "+ failB+ ")";
      LOG.info("\n\n-------------------------------------------\n" + "Beginning test, failing at " + injectionStr + "\n"+ "-------------------------------------------\n\n");
      MiniJournalCluster cluster=new MiniJournalCluster.Builder(conf).build();
      QuorumJournalManager qjm=null;
      try {
        qjm=createInjectableQJM(cluster);
        qjm.format(FAKE_NSINFO);
        List loggers=qjm.getLoggerSetForTests().getLoggersForTests();
        failIpcNumber(loggers.get(0),failA);
        failIpcNumber(loggers.get(1),failB);
        int lastAckedTxn=doWorkload(cluster,qjm);
        if (lastAckedTxn < 6) {
          LOG.info("Failed after injecting failures at " + injectionStr + ". This is expected since we injected a failure in the "+ "majority.");
        }
        qjm.close();
        qjm=null;
        qjm=createInjectableQJM(cluster);
        long lastRecoveredTxn=QJMTestUtil.recoverAndReturnLastTxn(qjm);
        assertTrue(lastRecoveredTxn >= lastAckedTxn);
        writeSegment(cluster,qjm,lastRecoveredTxn + 1,3,true);
      }
 catch (      Throwable t) {
        throw new RuntimeException("Test failed with injection: " + injectionStr,t);
      }
 finally {
        cluster.shutdown();
        cluster=null;
        IOUtils.closeStream(qjm);
        qjm=null;
      }
    }
  }
}

APIUtilityVerifier IterativeVerifier BooleanVerifier 

/** 
 * Test case in which three JournalNodes randomly flip flop between
 * up and down states every time they get an RPC.
 * The writer keeps track of the latest ACKed edit, and on every
 * recovery operation, ensures that it recovers at least to that
 * point or higher. Since at any given point, a majority of JNs
 * may be injecting faults, any writer operation is allowed to fail,
 * so long as the exception message indicates it failed due to injected
 * faults.
 * Given a random seed, the test should be entirely deterministic.
 */
@Test public void testRandomized() throws Exception {
  long seed;
  Long userSpecifiedSeed=Long.getLong(RAND_SEED_PROPERTY);
  if (userSpecifiedSeed != null) {
    LOG.info("Using seed specified in system property");
    seed=userSpecifiedSeed;
    ((Log4JLogger)ProtobufRpcEngine.LOG).getLogger().setLevel(Level.ALL);
  }
 else {
    seed=new Random().nextLong();
  }
  LOG.info("Random seed: " + seed);
  Random r=new Random(seed);
  MiniJournalCluster cluster=new MiniJournalCluster.Builder(conf).build();
  QuorumJournalManager qjmForInitialFormat=createInjectableQJM(cluster);
  qjmForInitialFormat.format(FAKE_NSINFO);
  qjmForInitialFormat.close();
  try {
    long txid=0;
    long lastAcked=0;
    for (int i=0; i < NUM_WRITER_ITERS; i++) {
      LOG.info("Starting writer " + i + "\n-------------------");
      QuorumJournalManager qjm=createRandomFaultyQJM(cluster,r);
      try {
        long recovered;
        try {
          recovered=QJMTestUtil.recoverAndReturnLastTxn(qjm);
        }
 catch (        Throwable t) {
          LOG.info("Failed recovery",t);
          checkException(t);
          continue;
        }
        assertTrue("Recovered only up to txnid " + recovered + " but had gotten an ack for "+ lastAcked,recovered >= lastAcked);
        txid=recovered + 1;
        if (txid > 100 && i % 10 == 1) {
          qjm.purgeLogsOlderThan(txid - 100);
        }
        Holder thrown=new Holder(null);
        for (int j=0; j < SEGMENTS_PER_WRITER; j++) {
          lastAcked=writeSegmentUntilCrash(cluster,qjm,txid,4,thrown);
          if (thrown.held != null) {
            LOG.info("Failed write",thrown.held);
            checkException(thrown.held);
            break;
          }
          txid+=4;
        }
      }
  finally {
        qjm.close();
      }
    }
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.qjournal.client.TestQuorumCall

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

@Test(timeout=10000) public void testQuorums() throws Exception {
  Map> futures=ImmutableMap.of("f1",SettableFuture.create(),"f2",SettableFuture.create(),"f3",SettableFuture.create());
  QuorumCall q=QuorumCall.create(futures);
  assertEquals(0,q.countResponses());
  futures.get("f1").set("first future");
  q.waitFor(1,0,0,100000,"test");
  q.waitFor(0,1,0,100000,"test");
  assertEquals(1,q.countResponses());
  futures.get("f2").setException(new Exception("error"));
  assertEquals(2,q.countResponses());
  futures.get("f3").set("second future");
  q.waitFor(3,0,100,100000,"test");
  q.waitFor(0,2,100,100000,"test");
  assertEquals(3,q.countResponses());
  assertEquals("f1=first future,f3=second future",Joiner.on(",").withKeyValueSeparator("=").join(new TreeMap(q.getResults())));
  try {
    q.waitFor(0,4,100,10,"test");
    fail("Didn't time out waiting for more responses than came back");
  }
 catch (  TimeoutException te) {
  }
}

Class: org.apache.hadoop.hdfs.qjournal.client.TestQuorumJournalManager

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testReaderWhileAnotherWrites() throws Exception {
  QuorumJournalManager readerQjm=closeLater(createSpyingQJM());
  List streams=Lists.newArrayList();
  readerQjm.selectInputStreams(streams,0,false);
  assertEquals(0,streams.size());
  writeSegment(cluster,qjm,1,3,true);
  readerQjm.selectInputStreams(streams,0,false);
  try {
    assertEquals(1,streams.size());
    EditLogInputStream stream=streams.get(0);
    assertEquals(1,stream.getFirstTxId());
    assertEquals(3,stream.getLastTxId());
    verifyEdits(streams,1,3);
    assertNull(stream.readOp());
  }
  finally {
    IOUtils.cleanup(LOG,streams.toArray(new Closeable[0]));
    streams.clear();
  }
  writeSegment(cluster,qjm,4,3,false);
  readerQjm.selectInputStreams(streams,0,false);
  try {
    assertEquals(1,streams.size());
    EditLogInputStream stream=streams.get(0);
    assertEquals(1,stream.getFirstTxId());
    assertEquals(3,stream.getLastTxId());
    verifyEdits(streams,1,3);
  }
  finally {
    IOUtils.cleanup(LOG,streams.toArray(new Closeable[0]));
    streams.clear();
  }
  qjm.finalizeLogSegment(4,6);
  readerQjm.selectInputStreams(streams,0,false);
  try {
    assertEquals(2,streams.size());
    assertEquals(4,streams.get(1).getFirstTxId());
    assertEquals(6,streams.get(1).getLastTxId());
    verifyEdits(streams,1,6);
  }
  finally {
    IOUtils.cleanup(LOG,streams.toArray(new Closeable[0]));
    streams.clear();
  }
}

APIUtilityVerifier BooleanVerifier 

@Test(timeout=20000) public void testCrashBetweenSyncLogAndPersistPaxosData() throws Exception {
  JournalFaultInjector faultInjector=JournalFaultInjector.instance=Mockito.mock(JournalFaultInjector.class);
  setupLoggers345();
  qjm=createSpyingQJM();
  spies=qjm.getLoggerSetForTests().getLoggersForTests();
  cluster.getJournalNode(2).stopAndJoin(0);
  injectIOE().when(spies.get(1)).acceptRecovery(Mockito.any(),Mockito.any());
  tryRecoveryExpectingFailure();
  cluster.restartJournalNode(2);
  qjm=createSpyingQJM();
  spies=qjm.getLoggerSetForTests().getLoggersForTests();
  injectIOE().when(spies.get(0)).prepareRecovery(Mockito.eq(1L));
  Mockito.doThrow(new IOException("Injected")).when(faultInjector).beforePersistPaxosData();
  tryRecoveryExpectingFailure();
  Mockito.reset(faultInjector);
  cluster.getJournalNode(2).stopAndJoin(0);
  qjm=createSpyingQJM();
  try {
    long recovered=QJMTestUtil.recoverAndReturnLastTxn(qjm);
    assertTrue(recovered >= 4);
  }
  finally {
    qjm.close();
  }
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

/** 
 * Test the case where one of the loggers misses a finalizeLogSegment()
 * call, and then misses the next startLogSegment() call before coming
 * back to life.
 * Previously, this caused it to keep on writing to the old log segment,
 * such that one logger had eg edits_1-10 while the others had edits_1-5 and
 * edits_6-10. This caused recovery to fail in certain cases.
 */
@Test public void testMissFinalizeAndNextStart() throws Exception {
  futureThrows(new IOException("injected")).when(spies.get(0)).finalizeLogSegment(Mockito.eq(1L),Mockito.eq(3L));
  futureThrows(new IOException("injected")).when(spies.get(0)).startLogSegment(Mockito.eq(4L),Mockito.eq(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION));
  failLoggerAtTxn(spies.get(1),4L);
  writeSegment(cluster,qjm,1,3,true);
  EditLogOutputStream stm=qjm.startLogSegment(4,NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  try {
    writeTxns(stm,4,1);
    fail("Did not fail to write");
  }
 catch (  QuorumException qe) {
    GenericTestUtils.assertExceptionContains("Writer out of sync",qe);
  }
 finally {
    stm.abort();
    qjm.close();
  }
  cluster.getJournalNode(2).stopAndJoin(0);
  qjm=createSpyingQJM();
  long recovered=QJMTestUtil.recoverAndReturnLastTxn(qjm);
  assertEquals(3L,recovered);
}

Class: org.apache.hadoop.hdfs.qjournal.server.TestJournalNode

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=100000) public void testHttpServer() throws Exception {
  String urlRoot=jn.getHttpServerURI();
  String pageContents=DFSTestUtil.urlGet(new URL(urlRoot + "/jmx"));
  assertTrue("Bad contents: " + pageContents,pageContents.contains("Hadoop:service=JournalNode,name=JvmMetrics"));
  byte[] EDITS_DATA=QJMTestUtil.createTxnData(1,3);
  IPCLoggerChannel ch=new IPCLoggerChannel(conf,FAKE_NSINFO,journalId,jn.getBoundIpcAddress());
  ch.newEpoch(1).get();
  ch.setEpoch(1);
  ch.startLogSegment(1,NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION).get();
  ch.sendEdits(1L,1,3,EDITS_DATA).get();
  ch.finalizeLogSegment(1,3).get();
  byte[] retrievedViaHttp=DFSTestUtil.urlGetBytes(new URL(urlRoot + "/getJournal?segmentTxId=1&jid=" + journalId));
  byte[] expected=Bytes.concat(Ints.toByteArray(HdfsConstants.NAMENODE_LAYOUT_VERSION),(new byte[]{0,0,0,0}),EDITS_DATA);
  assertArrayEquals(expected,retrievedViaHttp);
  URL badUrl=new URL(urlRoot + "/getJournal?segmentTxId=12345&jid=" + journalId);
  HttpURLConnection connection=(HttpURLConnection)badUrl.openConnection();
  try {
    assertEquals(404,connection.getResponseCode());
  }
  finally {
    connection.disconnect();
  }
}

Class: org.apache.hadoop.hdfs.qjournal.server.TestJournalNodeMXBean

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testJournalNodeMXBean() throws Exception {
  MBeanServer mbs=ManagementFactory.getPlatformMBeanServer();
  ObjectName mxbeanName=new ObjectName("Hadoop:service=JournalNode,name=JournalNodeInfo");
  String journalStatus=(String)mbs.getAttribute(mxbeanName,"JournalsStatus");
  assertEquals(jn.getJournalsStatus(),journalStatus);
  assertFalse(journalStatus.contains(NAMESERVICE));
  final NamespaceInfo FAKE_NSINFO=new NamespaceInfo(12345,"mycluster","my-bp",0L);
  jn.getOrCreateJournal(NAMESERVICE).format(FAKE_NSINFO);
  journalStatus=(String)mbs.getAttribute(mxbeanName,"JournalsStatus");
  assertEquals(jn.getJournalsStatus(),journalStatus);
  Map> jMap=new HashMap>();
  Map infoMap=new HashMap();
  infoMap.put("Formatted","true");
  jMap.put(NAMESERVICE,infoMap);
  assertEquals(JSON.toString(jMap),journalStatus);
  jCluster=new MiniJournalCluster.Builder(new Configuration()).format(false).numJournalNodes(NUM_JN).build();
  jn=jCluster.getJournalNode(0);
  journalStatus=(String)mbs.getAttribute(mxbeanName,"JournalsStatus");
  assertEquals(jn.getJournalsStatus(),journalStatus);
  jMap=new HashMap>();
  infoMap=new HashMap();
  infoMap.put("Formatted","true");
  jMap.put(NAMESERVICE,infoMap);
  assertEquals(JSON.toString(jMap),journalStatus);
}

Class: org.apache.hadoop.hdfs.security.TestDelegationToken

APIUtilityVerifier InternalCallVerifier IdentityVerifier EqualityVerifier HybridVerifier 

@Test public void testDelegationTokenWebHdfsApi() throws Exception {
  ((Log4JLogger)NamenodeWebHdfsMethods.LOG).getLogger().setLevel(Level.ALL);
  final String uri=WebHdfsFileSystem.SCHEME + "://" + config.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
  final UserGroupInformation ugi=UserGroupInformation.createUserForTesting("JobTracker",new String[]{"user"});
  final WebHdfsFileSystem webhdfs=ugi.doAs(new PrivilegedExceptionAction(){
    @Override public WebHdfsFileSystem run() throws Exception {
      return (WebHdfsFileSystem)FileSystem.get(new URI(uri),config);
    }
  }
);
{
    Credentials creds=new Credentials();
    final Token tokens[]=webhdfs.addDelegationTokens("JobTracker",creds);
    Assert.assertEquals(1,tokens.length);
    Assert.assertEquals(1,creds.numberOfTokens());
    Assert.assertSame(tokens[0],creds.getAllTokens().iterator().next());
    checkTokenIdentifier(ugi,tokens[0]);
    final Token tokens2[]=webhdfs.addDelegationTokens("JobTracker",creds);
    Assert.assertEquals(0,tokens2.length);
  }
}

Class: org.apache.hadoop.hdfs.security.TestDelegationTokenForProxyUser

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=5000) public void testWebHdfsDoAs() throws Exception {
  WebHdfsTestUtil.LOG.info("START: testWebHdfsDoAs()");
  WebHdfsTestUtil.LOG.info("ugi.getShortUserName()=" + ugi.getShortUserName());
  final WebHdfsFileSystem webhdfs=WebHdfsTestUtil.getWebHdfsFileSystemAs(ugi,config,WebHdfsFileSystem.SCHEME);
  final Path root=new Path("/");
  cluster.getFileSystem().setPermission(root,new FsPermission((short)0777));
  Whitebox.setInternalState(webhdfs,"ugi",proxyUgi);
{
    Path responsePath=webhdfs.getHomeDirectory();
    WebHdfsTestUtil.LOG.info("responsePath=" + responsePath);
    Assert.assertEquals(webhdfs.getUri() + "/user/" + PROXY_USER,responsePath.toString());
  }
  final Path f=new Path("/testWebHdfsDoAs/a.txt");
{
    FSDataOutputStream out=webhdfs.create(f);
    out.write("Hello, webhdfs user!".getBytes());
    out.close();
    final FileStatus status=webhdfs.getFileStatus(f);
    WebHdfsTestUtil.LOG.info("status.getOwner()=" + status.getOwner());
    Assert.assertEquals(PROXY_USER,status.getOwner());
  }
{
    final FSDataOutputStream out=webhdfs.append(f);
    out.write("\nHello again!".getBytes());
    out.close();
    final FileStatus status=webhdfs.getFileStatus(f);
    WebHdfsTestUtil.LOG.info("status.getOwner()=" + status.getOwner());
    WebHdfsTestUtil.LOG.info("status.getLen()  =" + status.getLen());
    Assert.assertEquals(PROXY_USER,status.getOwner());
  }
}

Class: org.apache.hadoop.hdfs.security.token.block.TestBlockToken

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * This test writes a file and gets the block locations without closing the
 * file, and tests the block token in the last block. Block token is verified
 * by ensuring it is of correct kind.
 * @throws IOException
 * @throws InterruptedException
 */
@Test public void testBlockTokenInLastLocatedBlock() throws IOException, InterruptedException {
  Configuration conf=new HdfsConfiguration();
  conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY,true);
  conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,512);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  try {
    FileSystem fs=cluster.getFileSystem();
    String fileName="/testBlockTokenInLastLocatedBlock";
    Path filePath=new Path(fileName);
    FSDataOutputStream out=fs.create(filePath,(short)1);
    out.write(new byte[1000]);
    LocatedBlocks locatedBlocks=cluster.getNameNodeRpc().getBlockLocations(fileName,0,1000);
    while (locatedBlocks.getLastLocatedBlock() == null) {
      Thread.sleep(100);
      locatedBlocks=cluster.getNameNodeRpc().getBlockLocations(fileName,0,1000);
    }
    Token token=locatedBlocks.getLastLocatedBlock().getBlockToken();
    Assert.assertEquals(BlockTokenIdentifier.KIND_NAME,token.getKind());
    out.close();
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier IterativeVerifier BranchVerifier UtilityVerifier InternalCallVerifier AssumptionSetter EqualityVerifier HybridVerifier 

/** 
 * Test that fast repeated invocations of createClientDatanodeProtocolProxy
 * will not end up using up thousands of sockets. This is a regression test
 * for HDFS-1965.
 */
@Test public void testBlockTokenRpcLeak() throws Exception {
  Configuration conf=new Configuration();
  conf.set(HADOOP_SECURITY_AUTHENTICATION,"kerberos");
  UserGroupInformation.setConfiguration(conf);
  Assume.assumeTrue(FD_DIR.exists());
  BlockTokenSecretManager sm=new BlockTokenSecretManager(blockKeyUpdateInterval,blockTokenLifetime,0,"fake-pool",null);
  Token token=sm.generateToken(block3,EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));
  final Server server=createMockDatanode(sm,token,conf);
  server.start();
  final InetSocketAddress addr=NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId=DFSTestUtil.getLocalDatanodeID(addr.getPort());
  ExtendedBlock b=new ExtendedBlock("fake-pool",new Block(12345L));
  LocatedBlock fakeBlock=new LocatedBlock(b,new DatanodeInfo[0]);
  fakeBlock.setBlockToken(token);
  ClientDatanodeProtocol proxyToNoWhere=RPC.getProxy(ClientDatanodeProtocol.class,ClientDatanodeProtocol.versionID,new InetSocketAddress("1.1.1.1",1),UserGroupInformation.createRemoteUser("junk"),conf,NetUtils.getDefaultSocketFactory(conf));
  ClientDatanodeProtocol proxy=null;
  int fdsAtStart=countOpenFileDescriptors();
  try {
    long endTime=Time.now() + 3000;
    while (Time.now() < endTime) {
      proxy=DFSUtil.createClientDatanodeProtocolProxy(fakeDnId,conf,1000,false,fakeBlock);
      assertEquals(block3.getBlockId(),proxy.getReplicaVisibleLength(block3));
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
      LOG.info("Num open fds:" + countOpenFileDescriptors());
    }
    int fdsAtEnd=countOpenFileDescriptors();
    if (fdsAtEnd - fdsAtStart > 50) {
      fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
    }
  }
  finally {
    server.stop();
  }
  RPC.stopProxy(proxyToNoWhere);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testBlockTokenRpc() throws Exception {
  Configuration conf=new Configuration();
  conf.set(HADOOP_SECURITY_AUTHENTICATION,"kerberos");
  UserGroupInformation.setConfiguration(conf);
  BlockTokenSecretManager sm=new BlockTokenSecretManager(blockKeyUpdateInterval,blockTokenLifetime,0,"fake-pool",null);
  Token token=sm.generateToken(block3,EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));
  final Server server=createMockDatanode(sm,token,conf);
  server.start();
  final InetSocketAddress addr=NetUtils.getConnectAddress(server);
  final UserGroupInformation ticket=UserGroupInformation.createRemoteUser(block3.toString());
  ticket.addToken(token);
  ClientDatanodeProtocol proxy=null;
  try {
    proxy=DFSUtil.createClientDatanodeProtocolProxy(addr,ticket,conf,NetUtils.getDefaultSocketFactory(conf));
    assertEquals(block3.getBlockId(),proxy.getReplicaVisibleLength(block3));
  }
  finally {
    server.stop();
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
  }
}

Class: org.apache.hadoop.hdfs.server.balancer.TestBalancer

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=100000) public void testUnknownDatanode() throws Exception {
  Configuration conf=new HdfsConfiguration();
  initConf(conf);
  long distribution[]=new long[]{50 * CAPACITY / 100,70 * CAPACITY / 100,0 * CAPACITY / 100};
  long capacities[]=new long[]{CAPACITY,CAPACITY,CAPACITY};
  String racks[]=new String[]{RACK0,RACK1,RACK1};
  int numDatanodes=distribution.length;
  if (capacities.length != numDatanodes || racks.length != numDatanodes) {
    throw new IllegalArgumentException("Array length is not the same");
  }
  final long totalUsedSpace=sum(distribution);
  ExtendedBlock[] blocks=generateBlocks(conf,totalUsedSpace,(short)numDatanodes);
  Block[][] blocksDN=distributeBlocks(blocks,(short)(numDatanodes - 1),distribution);
  conf.set(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY,"0.0f");
  cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).racks(racks).simulatedCapacities(capacities).build();
  try {
    cluster.waitActive();
    client=NameNodeProxies.createProxy(conf,cluster.getFileSystem(0).getUri(),ClientProtocol.class).getProxy();
    for (int i=0; i < 3; i++) {
      cluster.injectBlocks(i,Arrays.asList(blocksDN[i]),null);
    }
    cluster.startDataNodes(conf,1,true,null,new String[]{RACK0},null,new long[]{CAPACITY});
    cluster.triggerHeartbeats();
    Collection namenodes=DFSUtil.getNsServiceRpcUris(conf);
    Set datanodes=new HashSet();
    datanodes.add(cluster.getDataNodes().get(0).getDatanodeId().getHostName());
    Balancer.Parameters p=new Balancer.Parameters(Balancer.Parameters.DEFAULT.policy,Balancer.Parameters.DEFAULT.threshold,datanodes,Balancer.Parameters.DEFAULT.nodesToBeIncluded);
    final int r=Balancer.run(namenodes,p,conf);
    assertEquals(ExitStatus.SUCCESS.getExitCode(),r);
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.balancer.TestBalancerWithHANameNodes

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Test a cluster with even distribution, then a new empty node is added to
 * the cluster. Test start a cluster with specified number of nodes, and fills
 * it to be 30% full (with a single file replicated identically to all
 * datanodes); It then adds one new empty node and starts balancing.
 */
@Test(timeout=60000) public void testBalancerWithHANameNodes() throws Exception {
  Configuration conf=new HdfsConfiguration();
  TestBalancer.initConf(conf);
  long newNodeCapacity=TestBalancer.CAPACITY;
  String newNodeRack=TestBalancer.RACK2;
  String[] racks=new String[]{TestBalancer.RACK0,TestBalancer.RACK1};
  long[] capacities=new long[]{TestBalancer.CAPACITY,TestBalancer.CAPACITY};
  assertEquals(capacities.length,racks.length);
  int numOfDatanodes=capacities.length;
  NNConf nn1Conf=new MiniDFSNNTopology.NNConf("nn1");
  nn1Conf.setIpcPort(NameNode.DEFAULT_PORT);
  Configuration copiedConf=new Configuration(conf);
  cluster=new MiniDFSCluster.Builder(copiedConf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(capacities.length).racks(racks).simulatedCapacities(capacities).build();
  HATestUtil.setFailoverConfigurations(cluster,conf);
  try {
    cluster.waitActive();
    cluster.transitionToActive(1);
    Thread.sleep(500);
    client=NameNodeProxies.createProxy(conf,FileSystem.getDefaultUri(conf),ClientProtocol.class).getProxy();
    long totalCapacity=TestBalancer.sum(capacities);
    long totalUsedSpace=totalCapacity * 3 / 10;
    TestBalancer.createFile(cluster,TestBalancer.filePath,totalUsedSpace / numOfDatanodes,(short)numOfDatanodes,1);
    cluster.startDataNodes(conf,1,true,null,new String[]{newNodeRack},new long[]{newNodeCapacity});
    totalCapacity+=newNodeCapacity;
    TestBalancer.waitForHeartBeat(totalUsedSpace,totalCapacity,client,cluster);
    Collection namenodes=DFSUtil.getNsServiceRpcUris(conf);
    assertEquals(1,namenodes.size());
    assertTrue(namenodes.contains(HATestUtil.getLogicalUri(cluster)));
    final int r=Balancer.run(namenodes,Balancer.Parameters.DEFAULT,conf);
    assertEquals(ExitStatus.SUCCESS.getExitCode(),r);
    TestBalancer.waitForBalancer(totalUsedSpace,totalCapacity,client,cluster,Balancer.Parameters.DEFAULT);
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.balancer.TestBalancerWithNodeGroup

APIUtilityVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier 

/** 
 * Create a cluster with even distribution, and a new empty node is added to
 * the cluster, then test rack locality for balancer policy. 
 */
@Test(timeout=60000) public void testBalancerWithRackLocality() throws Exception {
  Configuration conf=createConf();
  long[] capacities=new long[]{CAPACITY,CAPACITY};
  String[] racks=new String[]{RACK0,RACK1};
  String[] nodeGroups=new String[]{NODEGROUP0,NODEGROUP1};
  int numOfDatanodes=capacities.length;
  assertEquals(numOfDatanodes,racks.length);
  MiniDFSCluster.Builder builder=new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length).racks(racks).simulatedCapacities(capacities);
  MiniDFSClusterWithNodeGroup.setNodeGroups(nodeGroups);
  cluster=new MiniDFSClusterWithNodeGroup(builder);
  try {
    cluster.waitActive();
    client=NameNodeProxies.createProxy(conf,cluster.getFileSystem(0).getUri(),ClientProtocol.class).getProxy();
    long totalCapacity=TestBalancer.sum(capacities);
    long totalUsedSpace=totalCapacity * 3 / 10;
    long length=totalUsedSpace / numOfDatanodes;
    TestBalancer.createFile(cluster,filePath,length,(short)numOfDatanodes,0);
    LocatedBlocks lbs=client.getBlockLocations(filePath.toUri().getPath(),0,length);
    Set before=getBlocksOnRack(lbs.getLocatedBlocks(),RACK0);
    long newCapacity=CAPACITY;
    String newRack=RACK1;
    String newNodeGroup=NODEGROUP2;
    cluster.startDataNodes(conf,1,true,null,new String[]{newRack},new long[]{newCapacity},new String[]{newNodeGroup});
    totalCapacity+=newCapacity;
    runBalancerCanFinish(conf,totalUsedSpace,totalCapacity);
    lbs=client.getBlockLocations(filePath.toUri().getPath(),0,length);
    Set after=getBlocksOnRack(lbs.getLocatedBlocks(),RACK0);
    assertEquals(before,after);
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.blockmanagement.TestBlockInfo

APIUtilityVerifier IterativeVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testBlockListMoveToHead() throws Exception {
  LOG.info("BlockInfo moveToHead tests...");
  final int MAX_BLOCKS=10;
  DatanodeStorageInfo dd=DFSTestUtil.createDatanodeStorageInfo("s1","1.1.1.1");
  ArrayList blockList=new ArrayList(MAX_BLOCKS);
  ArrayList blockInfoList=new ArrayList();
  int headIndex;
  int curIndex;
  LOG.info("Building block list...");
  for (int i=0; i < MAX_BLOCKS; i++) {
    blockList.add(new Block(i,0,GenerationStamp.LAST_RESERVED_STAMP));
    blockInfoList.add(new BlockInfo(blockList.get(i),3));
    dd.addBlock(blockInfoList.get(i));
    assertEquals("Find datanode should be 0",0,blockInfoList.get(i).findStorageInfo(dd));
  }
  LOG.info("Checking list length...");
  assertEquals("Length should be MAX_BLOCK",MAX_BLOCKS,dd.numBlocks());
  Iterator it=dd.getBlockIterator();
  int len=0;
  while (it.hasNext()) {
    it.next();
    len++;
  }
  assertEquals("There should be MAX_BLOCK blockInfo's",MAX_BLOCKS,len);
  headIndex=dd.getBlockListHeadForTesting().findStorageInfo(dd);
  LOG.info("Moving each block to the head of the list...");
  for (int i=0; i < MAX_BLOCKS; i++) {
    curIndex=blockInfoList.get(i).findStorageInfo(dd);
    headIndex=dd.moveBlockToHead(blockInfoList.get(i),curIndex,headIndex);
    assertEquals("Block should be at the head of the list now.",blockInfoList.get(i),dd.getBlockListHeadForTesting());
  }
  LOG.info("Moving head to the head...");
  BlockInfo temp=dd.getBlockListHeadForTesting();
  curIndex=0;
  headIndex=0;
  dd.moveBlockToHead(temp,curIndex,headIndex);
  assertEquals("Moving head to the head of the list shopuld not change the list",temp,dd.getBlockListHeadForTesting());
  LOG.info("Checking elements of the list...");
  temp=dd.getBlockListHeadForTesting();
  assertNotNull("Head should not be null",temp);
  int c=MAX_BLOCKS - 1;
  while (temp != null) {
    assertEquals("Expected element is not on the list",blockInfoList.get(c--),temp);
    temp=temp.getNext(0);
  }
  LOG.info("Moving random blocks to the head of the list...");
  headIndex=dd.getBlockListHeadForTesting().findStorageInfo(dd);
  Random rand=new Random();
  for (int i=0; i < MAX_BLOCKS; i++) {
    int j=rand.nextInt(MAX_BLOCKS);
    curIndex=blockInfoList.get(j).findStorageInfo(dd);
    headIndex=dd.moveBlockToHead(blockInfoList.get(j),curIndex,headIndex);
    assertEquals("Block should be at the head of the list now.",blockInfoList.get(j),dd.getBlockListHeadForTesting());
  }
}

Class: org.apache.hadoop.hdfs.server.blockmanagement.TestBlockManager

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testSafeModeIBR() throws Exception {
  DatanodeDescriptor node=spy(nodes.get(0));
  DatanodeStorageInfo ds=node.getStorageInfos()[0];
  node.setDatanodeUuidForTesting(ds.getStorageID());
  node.isAlive=true;
  DatanodeRegistration nodeReg=new DatanodeRegistration(node,null,null,"");
  doReturn(true).when(fsn).isInStartupSafeMode();
  bm.getDatanodeManager().registerDatanode(nodeReg);
  bm.getDatanodeManager().addDatanode(node);
  assertEquals(node,bm.getDatanodeManager().getDatanode(node));
  assertEquals(0,ds.getBlockReportCount());
  reset(node);
  bm.processReport(node,new DatanodeStorage(ds.getStorageID()),new BlockListAsLongs(null,null));
  assertEquals(1,ds.getBlockReportCount());
  reset(node);
  bm.processReport(node,new DatanodeStorage(ds.getStorageID()),new BlockListAsLongs(null,null));
  assertEquals(1,ds.getBlockReportCount());
  bm.getDatanodeManager().removeDatanode(node);
  reset(node);
  bm.getDatanodeManager().registerDatanode(nodeReg);
  verify(node).updateRegInfo(nodeReg);
  assertEquals(0,ds.getBlockReportCount());
  reset(node);
  bm.processReport(node,new DatanodeStorage(ds.getStorageID()),new BlockListAsLongs(null,null));
  assertEquals(1,ds.getBlockReportCount());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testSafeModeIBRAfterIncremental() throws Exception {
  DatanodeDescriptor node=spy(nodes.get(0));
  DatanodeStorageInfo ds=node.getStorageInfos()[0];
  node.setDatanodeUuidForTesting(ds.getStorageID());
  node.isAlive=true;
  DatanodeRegistration nodeReg=new DatanodeRegistration(node,null,null,"");
  doReturn(true).when(fsn).isInStartupSafeMode();
  bm.getDatanodeManager().registerDatanode(nodeReg);
  bm.getDatanodeManager().addDatanode(node);
  assertEquals(node,bm.getDatanodeManager().getDatanode(node));
  assertEquals(0,ds.getBlockReportCount());
  reset(node);
  doReturn(1).when(node).numBlocks();
  bm.processReport(node,new DatanodeStorage(ds.getStorageID()),new BlockListAsLongs(null,null));
  assertEquals(1,ds.getBlockReportCount());
}

Class: org.apache.hadoop.hdfs.server.blockmanagement.TestBlockTokenWithDFS

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testRead() throws Exception {
  MiniDFSCluster cluster=null;
  int numDataNodes=2;
  Configuration conf=getConf(numDataNodes);
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
    cluster.waitActive();
    assertEquals(numDataNodes,cluster.getDataNodes().size());
    final NameNode nn=cluster.getNameNode();
    final NamenodeProtocols nnProto=nn.getRpcServer();
    final BlockManager bm=nn.getNamesystem().getBlockManager();
    final BlockTokenSecretManager sm=bm.getBlockTokenSecretManager();
    SecurityTestUtil.setBlockTokenLifetime(sm,1000L);
    Path fileToRead=new Path(FILE_TO_READ);
    FileSystem fs=cluster.getFileSystem();
    createFile(fs,fileToRead);
    FSDataInputStream in1=fs.open(fileToRead);
    assertTrue(checkFile1(in1));
    FSDataInputStream in2=fs.open(fileToRead);
    assertTrue(checkFile1(in2));
    FSDataInputStream in3=fs.open(fileToRead);
    assertTrue(checkFile2(in3));
    DFSClient client=null;
    try {
      client=new DFSClient(new InetSocketAddress("localhost",cluster.getNameNodePort()),conf);
    }
  finally {
      if (client != null)       client.close();
    }
    List locatedBlocks=nnProto.getBlockLocations(FILE_TO_READ,0,FILE_SIZE).getLocatedBlocks();
    LocatedBlock lblock=locatedBlocks.get(0);
    Token myToken=lblock.getBlockToken();
    assertFalse(SecurityTestUtil.isBlockTokenExpired(myToken));
    tryRead(conf,lblock,true);
    while (!SecurityTestUtil.isBlockTokenExpired(myToken)) {
      try {
        Thread.sleep(10);
      }
 catch (      InterruptedException ignored) {
      }
    }
    assertTrue(SecurityTestUtil.isBlockTokenExpired(myToken));
    tryRead(conf,lblock,false);
    lblock.setBlockToken(sm.generateToken(lblock.getBlock(),EnumSet.of(BlockTokenSecretManager.AccessMode.READ)));
    tryRead(conf,lblock,true);
    ExtendedBlock wrongBlock=new ExtendedBlock(lblock.getBlock().getBlockPoolId(),lblock.getBlock().getBlockId() + 1);
    lblock.setBlockToken(sm.generateToken(wrongBlock,EnumSet.of(BlockTokenSecretManager.AccessMode.READ)));
    tryRead(conf,lblock,false);
    lblock.setBlockToken(sm.generateToken(lblock.getBlock(),EnumSet.of(BlockTokenSecretManager.AccessMode.WRITE,BlockTokenSecretManager.AccessMode.COPY,BlockTokenSecretManager.AccessMode.REPLACE)));
    tryRead(conf,lblock,false);
    SecurityTestUtil.setBlockTokenLifetime(sm,600 * 1000L);
    List lblocks=DFSTestUtil.getAllBlocks(in1);
    for (    LocatedBlock blk : lblocks) {
      assertTrue(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
    }
    in1.seek(0);
    assertTrue(checkFile1(in1));
    List lblocks2=DFSTestUtil.getAllBlocks(in2);
    for (    LocatedBlock blk : lblocks2) {
      assertTrue(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
    }
    assertTrue(in2.seekToNewSource(0));
    assertTrue(checkFile1(in2));
    List lblocks3=DFSTestUtil.getAllBlocks(in3);
    for (    LocatedBlock blk : lblocks3) {
      assertTrue(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
    }
    assertTrue(checkFile2(in3));
    assertTrue(cluster.restartDataNodes(true));
    cluster.waitActive();
    assertEquals(numDataNodes,cluster.getDataNodes().size());
    cluster.shutdownNameNode(0);
    lblocks=DFSTestUtil.getAllBlocks(in1);
    for (    LocatedBlock blk : lblocks) {
      assertFalse(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
    }
    in1.seek(0);
    assertTrue(checkFile1(in1));
    lblocks2=DFSTestUtil.getAllBlocks(in2);
    for (    LocatedBlock blk : lblocks2) {
      assertFalse(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
    }
    in2.seekToNewSource(0);
    assertTrue(checkFile1(in2));
    lblocks3=DFSTestUtil.getAllBlocks(in3);
    for (    LocatedBlock blk : lblocks3) {
      assertFalse(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
    }
    assertTrue(checkFile2(in3));
    cluster.restartNameNode(0);
    cluster.shutdownNameNode(0);
    in1.seek(0);
    assertTrue(checkFile1(in1));
    in2.seekToNewSource(0);
    assertTrue(checkFile1(in2));
    assertTrue(checkFile2(in3));
    cluster.restartNameNode(0);
    assertTrue(cluster.restartDataNodes(true));
    cluster.waitActive();
    assertEquals(numDataNodes,cluster.getDataNodes().size());
    cluster.shutdownNameNode(0);
    in1.seek(0);
    assertFalse(checkFile1(in1));
    assertFalse(checkFile2(in3));
    cluster.restartNameNode(0);
    in1.seek(0);
    assertTrue(checkFile1(in1));
    in2.seekToNewSource(0);
    assertTrue(checkFile1(in2));
    assertTrue(checkFile2(in3));
    assertTrue(cluster.restartDataNodes(false));
    cluster.waitActive();
    assertEquals(numDataNodes,cluster.getDataNodes().size());
    in1.seek(0);
    assertTrue(checkFile1(in1));
    in2.seekToNewSource(0);
    assertTrue(checkFile1(in2));
    assertTrue(checkFile2(in3));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * testing that APPEND operation can handle token expiration when
 * re-establishing pipeline is needed
 */
@Test public void testAppend() throws Exception {
  MiniDFSCluster cluster=null;
  int numDataNodes=2;
  Configuration conf=getConf(numDataNodes);
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
    cluster.waitActive();
    assertEquals(numDataNodes,cluster.getDataNodes().size());
    final NameNode nn=cluster.getNameNode();
    final BlockManager bm=nn.getNamesystem().getBlockManager();
    final BlockTokenSecretManager sm=bm.getBlockTokenSecretManager();
    SecurityTestUtil.setBlockTokenLifetime(sm,1000L);
    Path fileToAppend=new Path(FILE_TO_APPEND);
    FileSystem fs=cluster.getFileSystem();
    FSDataOutputStream stm=writeFile(fs,fileToAppend,(short)numDataNodes,BLOCK_SIZE);
    stm.write(rawData,0,1);
    stm.close();
    stm=fs.append(fileToAppend);
    int mid=rawData.length - 1;
    stm.write(rawData,1,mid - 1);
    stm.hflush();
    Token token=DFSTestUtil.getBlockToken(stm);
    while (!SecurityTestUtil.isBlockTokenExpired(token)) {
      try {
        Thread.sleep(10);
      }
 catch (      InterruptedException ignored) {
      }
    }
    cluster.stopDataNode(0);
    stm.write(rawData,mid,rawData.length - mid);
    stm.close();
    FSDataInputStream in5=fs.open(fileToAppend);
    assertTrue(checkFile1(in5));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * testing that WRITE operation can handle token expiration when
 * re-establishing pipeline is needed
 */
@Test public void testWrite() throws Exception {
  MiniDFSCluster cluster=null;
  int numDataNodes=2;
  Configuration conf=getConf(numDataNodes);
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
    cluster.waitActive();
    assertEquals(numDataNodes,cluster.getDataNodes().size());
    final NameNode nn=cluster.getNameNode();
    final BlockManager bm=nn.getNamesystem().getBlockManager();
    final BlockTokenSecretManager sm=bm.getBlockTokenSecretManager();
    SecurityTestUtil.setBlockTokenLifetime(sm,1000L);
    Path fileToWrite=new Path(FILE_TO_WRITE);
    FileSystem fs=cluster.getFileSystem();
    FSDataOutputStream stm=writeFile(fs,fileToWrite,(short)numDataNodes,BLOCK_SIZE);
    int mid=rawData.length - 1;
    stm.write(rawData,0,mid);
    stm.hflush();
    Token token=DFSTestUtil.getBlockToken(stm);
    while (!SecurityTestUtil.isBlockTokenExpired(token)) {
      try {
        Thread.sleep(10);
      }
 catch (      InterruptedException ignored) {
      }
    }
    cluster.stopDataNode(0);
    stm.write(rawData,mid,rawData.length - mid);
    stm.close();
    FSDataInputStream in4=fs.open(fileToWrite);
    assertTrue(checkFile1(in4));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.blockmanagement.TestBlocksWithNotEnoughRacks

APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

@Test public void testCorruptBlockRereplicatedAcrossRacks() throws Exception {
  Configuration conf=getConf();
  short REPLICATION_FACTOR=2;
  int fileLen=512;
  final Path filePath=new Path("/testFile");
  String racks[]={"/rack1","/rack1","/rack2","/rack2"};
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns=cluster.getNameNode().getNamesystem();
  try {
    final FileSystem fs=cluster.getFileSystem();
    DFSTestUtil.createFile(fs,filePath,fileLen,REPLICATION_FACTOR,1L);
    final String fileContent=DFSTestUtil.readFile(fs,filePath);
    ExtendedBlock b=DFSTestUtil.getFirstBlock(fs,filePath);
    DFSTestUtil.waitForReplication(cluster,b,2,REPLICATION_FACTOR,0);
    int dnToCorrupt=DFSTestUtil.firstDnWithBlock(cluster,b);
    assertTrue(MiniDFSCluster.corruptReplica(dnToCorrupt,b));
    cluster.restartDataNode(dnToCorrupt);
    DFSTestUtil.waitCorruptReplicas(fs,ns,filePath,b,1);
    DFSTestUtil.waitForReplication(cluster,b,2,REPLICATION_FACTOR,0);
    for (int i=0; i < racks.length; i++) {
      String blockContent=cluster.readBlockOnDataNode(i,b);
      if (blockContent != null && i != dnToCorrupt) {
        assertEquals("Corrupt replica",fileContent,blockContent);
      }
    }
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testNodeDecomissionWithOverreplicationRespectsRackPolicy() throws Exception {
  Configuration conf=getConf();
  short REPLICATION_FACTOR=5;
  final Path filePath=new Path("/testFile");
  FileSystem localFileSys=FileSystem.getLocal(conf);
  Path workingDir=localFileSys.getWorkingDirectory();
  Path dir=new Path(workingDir,"build/test/data/temp/decommission");
  Path excludeFile=new Path(dir,"exclude");
  Path includeFile=new Path(dir,"include");
  assertTrue(localFileSys.mkdirs(dir));
  DFSTestUtil.writeFile(localFileSys,excludeFile,"");
  DFSTestUtil.writeFile(localFileSys,includeFile,"");
  conf.set(DFSConfigKeys.DFS_HOSTS,includeFile.toUri().getPath());
  conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE,excludeFile.toUri().getPath());
  String racks[]={"/rack1","/rack2","/rack1","/rack1","/rack1"};
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns=cluster.getNameNode().getNamesystem();
  try {
    final FileSystem fs=cluster.getFileSystem();
    DFSTestUtil.createFile(fs,filePath,1L,REPLICATION_FACTOR,1L);
    ExtendedBlock b=DFSTestUtil.getFirstBlock(fs,filePath);
    DFSTestUtil.waitForReplication(cluster,b,2,REPLICATION_FACTOR,0);
    REPLICATION_FACTOR=2;
    fs.setReplication(filePath,REPLICATION_FACTOR);
    BlockLocation locs[]=fs.getFileBlockLocations(fs.getFileStatus(filePath),0,Long.MAX_VALUE);
    for (    String top : locs[0].getTopologyPaths()) {
      if (!top.startsWith("/rack2")) {
        String name=top.substring("/rack1".length() + 1);
        DFSTestUtil.writeFile(localFileSys,excludeFile,name);
        ns.getBlockManager().getDatanodeManager().refreshNodes(conf);
        DFSTestUtil.waitForDecommission(fs,name);
        break;
      }
    }
    DFSTestUtil.waitForReplication(cluster,b,2,REPLICATION_FACTOR,0);
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testNodeDecomissionRespectsRackPolicy() throws Exception {
  Configuration conf=getConf();
  short REPLICATION_FACTOR=2;
  final Path filePath=new Path("/testFile");
  FileSystem localFileSys=FileSystem.getLocal(conf);
  Path workingDir=localFileSys.getWorkingDirectory();
  Path dir=new Path(workingDir,"build/test/data/temp/decommission");
  Path excludeFile=new Path(dir,"exclude");
  Path includeFile=new Path(dir,"include");
  assertTrue(localFileSys.mkdirs(dir));
  DFSTestUtil.writeFile(localFileSys,excludeFile,"");
  DFSTestUtil.writeFile(localFileSys,includeFile,"");
  conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE,excludeFile.toUri().getPath());
  conf.set(DFSConfigKeys.DFS_HOSTS,includeFile.toUri().getPath());
  String racks[]={"/rack1","/rack1","/rack2","/rack2"};
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns=cluster.getNameNode().getNamesystem();
  try {
    final FileSystem fs=cluster.getFileSystem();
    DFSTestUtil.createFile(fs,filePath,1L,REPLICATION_FACTOR,1L);
    ExtendedBlock b=DFSTestUtil.getFirstBlock(fs,filePath);
    DFSTestUtil.waitForReplication(cluster,b,2,REPLICATION_FACTOR,0);
    BlockLocation locs[]=fs.getFileBlockLocations(fs.getFileStatus(filePath),0,Long.MAX_VALUE);
    String name=locs[0].getNames()[0];
    DFSTestUtil.writeFile(localFileSys,excludeFile,name);
    ns.getBlockManager().getDatanodeManager().refreshNodes(conf);
    DFSTestUtil.waitForDecommission(fs,name);
    DFSTestUtil.waitForReplication(cluster,b,2,REPLICATION_FACTOR,0);
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier 

@Test public void testReduceReplFactorDueToRejoinRespectsRackPolicy() throws Exception {
  Configuration conf=getConf();
  short REPLICATION_FACTOR=2;
  final Path filePath=new Path("/testFile");
  String racks[]={"/rack1","/rack1","/rack2"};
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns=cluster.getNameNode().getNamesystem();
  final DatanodeManager dm=ns.getBlockManager().getDatanodeManager();
  try {
    final FileSystem fs=cluster.getFileSystem();
    DFSTestUtil.createFile(fs,filePath,1L,REPLICATION_FACTOR,1L);
    ExtendedBlock b=DFSTestUtil.getFirstBlock(fs,filePath);
    DFSTestUtil.waitForReplication(cluster,b,2,REPLICATION_FACTOR,0);
    ArrayList datanodes=cluster.getDataNodes();
    assertEquals(3,datanodes.size());
    DataNode dataNode=datanodes.get(2);
    DatanodeID dnId=dataNode.getDatanodeId();
    cluster.stopDataNode(2);
    dm.removeDatanode(dnId);
    DFSTestUtil.waitForReplication(cluster,b,1,REPLICATION_FACTOR,1);
    String rack2[]={"/rack2"};
    cluster.startDataNodes(conf,1,true,null,rack2);
    cluster.waitActive();
    DFSTestUtil.waitForReplication(cluster,b,2,REPLICATION_FACTOR,0);
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.blockmanagement.TestOverReplicatedBlocks

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test processOverReplicatedBlock can handle corrupt replicas fine.
 * It make sure that it won't treat corrupt replicas as valid ones 
 * thus prevents NN deleting valid replicas but keeping
 * corrupt ones.
 */
@Test public void testProcesOverReplicateBlock() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,1000L);
  conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY,Integer.toString(2));
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
  FileSystem fs=cluster.getFileSystem();
  try {
    final Path fileName=new Path("/foo1");
    DFSTestUtil.createFile(fs,fileName,2,(short)3,0L);
    DFSTestUtil.waitReplication(fs,fileName,(short)3);
    ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,fileName);
    assertTrue(TestDatanodeBlockScanner.corruptReplica(block,0));
    DataNodeProperties dnProps=cluster.stopDataNode(0);
    File scanLog=new File(MiniDFSCluster.getFinalizedDir(cluster.getInstanceStorageDir(0,0),cluster.getNamesystem().getBlockPoolId()).getParent().toString() + "/../dncp_block_verification.log.prev");
    for (int i=0; !scanLog.delete(); i++) {
      assertTrue("Could not delete log file in one minute",i < 60);
      try {
        Thread.sleep(1000);
      }
 catch (      InterruptedException ignored) {
      }
    }
    cluster.restartDataNode(dnProps);
    DFSTestUtil.waitReplication(fs,fileName,(short)2);
    String blockPoolId=cluster.getNamesystem().getBlockPoolId();
    final DatanodeID corruptDataNode=DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(2),blockPoolId);
    final FSNamesystem namesystem=cluster.getNamesystem();
    final BlockManager bm=namesystem.getBlockManager();
    final HeartbeatManager hm=bm.getDatanodeManager().getHeartbeatManager();
    try {
      namesystem.writeLock();
synchronized (hm) {
        String corruptMachineName=corruptDataNode.getXferAddr();
        for (        DatanodeDescriptor datanode : hm.getDatanodes()) {
          if (!corruptMachineName.equals(datanode.getXferAddr())) {
            datanode.getStorageInfos()[0].setUtilizationForTesting(100L,100L,0,100L);
            datanode.updateHeartbeat(BlockManagerTestUtil.getStorageReportsForDatanode(datanode),0L,0L,0,0);
          }
        }
        NameNodeAdapter.setReplication(namesystem,fileName.toString(),(short)1);
        assertEquals(1,bm.countNodes(block.getLocalBlock()).liveReplicas());
      }
    }
  finally {
      namesystem.writeUnlock();
    }
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test over replicated block should get invalidated when decreasing the
 * replication for a partial block.
 */
@Test public void testInvalidateOverReplicatedBlock() throws Exception {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
  try {
    final FSNamesystem namesystem=cluster.getNamesystem();
    final BlockManager bm=namesystem.getBlockManager();
    FileSystem fs=cluster.getFileSystem();
    Path p=new Path(MiniDFSCluster.getBaseDirectory(),"/foo1");
    FSDataOutputStream out=fs.create(p,(short)2);
    out.writeBytes("HDFS-3119: " + p);
    out.hsync();
    fs.setReplication(p,(short)1);
    out.close();
    ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,p);
    assertEquals("Expected only one live replica for the block",1,bm.countNodes(block.getLocalBlock()).liveReplicas());
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * The test verifies that replica for deletion is chosen on a node,
 * with the oldest heartbeat, when this heartbeat is larger than the
 * tolerable heartbeat interval.
 * It creates a file with several blocks and replication 4.
 * The last DN is configured to send heartbeats rarely.
 * Test waits until the tolerable heartbeat interval expires, and reduces
 * replication of the file. All replica deletions should be scheduled for the
 * last node. No replicas will actually be deleted, since last DN doesn't
 * send heartbeats. 
 */
@Test public void testChooseReplicaToDelete() throws Exception {
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    Configuration conf=new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,SMALL_BLOCK_SIZE);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    fs=cluster.getFileSystem();
    final FSNamesystem namesystem=cluster.getNamesystem();
    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,300);
    cluster.startDataNodes(conf,1,true,null,null,null);
    DataNode lastDN=cluster.getDataNodes().get(3);
    DatanodeRegistration dnReg=DataNodeTestUtils.getDNRegistrationForBP(lastDN,namesystem.getBlockPoolId());
    String lastDNid=dnReg.getDatanodeUuid();
    final Path fileName=new Path("/foo2");
    DFSTestUtil.createFile(fs,fileName,SMALL_FILE_LENGTH,(short)4,0L);
    DFSTestUtil.waitReplication(fs,fileName,(short)4);
    DatanodeDescriptor nodeInfo=null;
    long lastHeartbeat=0;
    long waitTime=DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT * 1000 * (DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT + 1);
    do {
      nodeInfo=namesystem.getBlockManager().getDatanodeManager().getDatanode(dnReg);
      lastHeartbeat=nodeInfo.getLastUpdate();
    }
 while (now() - lastHeartbeat < waitTime);
    fs.setReplication(fileName,(short)3);
    BlockLocation locs[]=fs.getFileBlockLocations(fs.getFileStatus(fileName),0,Long.MAX_VALUE);
    namesystem.readLock();
    Collection dnBlocks=namesystem.getBlockManager().excessReplicateMap.get(lastDNid);
    assertEquals("Replicas on node " + lastDNid + " should have been deleted",SMALL_FILE_LENGTH / SMALL_BLOCK_SIZE,dnBlocks.size());
    namesystem.readUnlock();
    for (    BlockLocation location : locs)     assertEquals("Block should still have 4 replicas",4,location.getNames().length);
  }
  finally {
    if (fs != null)     fs.close();
    if (cluster != null)     cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.blockmanagement.TestPendingReplication

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test if DatanodeProtocol#blockReceivedAndDeleted can correctly update the
 * pending replications. Also make sure the blockReceivedAndDeleted call is
 * idempotent to the pending replications. 
 */
@Test public void testBlockReceived() throws Exception {
  final Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,1024);
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(DATANODE_COUNT).build();
    cluster.waitActive();
    DistributedFileSystem hdfs=cluster.getFileSystem();
    FSNamesystem fsn=cluster.getNamesystem();
    BlockManager blkManager=fsn.getBlockManager();
    final String file="/tmp.txt";
    final Path filePath=new Path(file);
    short replFactor=1;
    DFSTestUtil.createFile(hdfs,filePath,1024L,replFactor,0);
    ArrayList datanodes=cluster.getDataNodes();
    for (int i=0; i < DATANODE_COUNT; i++) {
      DataNodeTestUtils.setHeartbeatsDisabledForTests(datanodes.get(i),true);
    }
    hdfs.setReplication(filePath,(short)DATANODE_COUNT);
    BlockManagerTestUtil.computeAllPendingWork(blkManager);
    assertEquals(1,blkManager.pendingReplications.size());
    INodeFile fileNode=fsn.getFSDirectory().getINode4Write(file).asFile();
    Block[] blocks=fileNode.getBlocks();
    assertEquals(DATANODE_COUNT - 1,blkManager.pendingReplications.getNumReplicas(blocks[0]));
    LocatedBlock locatedBlock=hdfs.getClient().getLocatedBlocks(file,0).get(0);
    DatanodeInfo existingDn=(locatedBlock.getLocations())[0];
    int reportDnNum=0;
    String poolId=cluster.getNamesystem().getBlockPoolId();
    for (int i=0; i < DATANODE_COUNT && reportDnNum < 2; i++) {
      if (!datanodes.get(i).getDatanodeId().equals(existingDn)) {
        DatanodeRegistration dnR=datanodes.get(i).getDNRegistrationForBP(poolId);
        StorageReceivedDeletedBlocks[] report={new StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored",new ReceivedDeletedBlockInfo[]{new ReceivedDeletedBlockInfo(blocks[0],BlockStatus.RECEIVED_BLOCK,"")})};
        cluster.getNameNodeRpc().blockReceivedAndDeleted(dnR,poolId,report);
        reportDnNum++;
      }
    }
    assertEquals(DATANODE_COUNT - 3,blkManager.pendingReplications.getNumReplicas(blocks[0]));
    for (int i=0; i < DATANODE_COUNT && reportDnNum < 2; i++) {
      if (!datanodes.get(i).getDatanodeId().equals(existingDn)) {
        DatanodeRegistration dnR=datanodes.get(i).getDNRegistrationForBP(poolId);
        StorageReceivedDeletedBlocks[] report={new StorageReceivedDeletedBlocks("Fake-storage-ID-Ignored",new ReceivedDeletedBlockInfo[]{new ReceivedDeletedBlockInfo(blocks[0],BlockStatus.RECEIVED_BLOCK,"")})};
        cluster.getNameNodeRpc().blockReceivedAndDeleted(dnR,poolId,report);
        reportDnNum++;
      }
    }
    assertEquals(DATANODE_COUNT - 3,blkManager.pendingReplications.getNumReplicas(blocks[0]));
    for (int i=0; i < DATANODE_COUNT; i++) {
      DataNodeTestUtils.setHeartbeatsDisabledForTests(datanodes.get(i),false);
      DataNodeTestUtils.triggerHeartbeat(datanodes.get(i));
    }
    Thread.sleep(5000);
    assertEquals(0,blkManager.pendingReplications.size());
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.blockmanagement.TestRBWBlockInvalidation

APIUtilityVerifier InternalCallVerifier BooleanVerifier AssumptionSetter EqualityVerifier HybridVerifier 

/** 
 * Test when a block's replica is removed from RBW folder in one of the
 * datanode, namenode should ask to invalidate that corrupted block and
 * schedule replication for one more replica for that under replicated block.
 */
@Test(timeout=600000) public void testBlockInvalidationWhenRBWReplicaMissedInDN() throws IOException, InterruptedException {
  assumeTrue(!Path.WINDOWS);
  Configuration conf=new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY,2);
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,300);
  conf.setLong(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,1);
  conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,1);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  FSDataOutputStream out=null;
  try {
    final FSNamesystem namesystem=cluster.getNamesystem();
    FileSystem fs=cluster.getFileSystem();
    Path testPath=new Path("/tmp/TestRBWBlockInvalidation","foo1");
    out=fs.create(testPath,(short)2);
    out.writeBytes("HDFS-3157: " + testPath);
    out.hsync();
    cluster.startDataNodes(conf,1,true,null,null,null);
    String bpid=namesystem.getBlockPoolId();
    ExtendedBlock blk=DFSTestUtil.getFirstBlock(fs,testPath);
    Block block=blk.getLocalBlock();
    DataNode dn=cluster.getDataNodes().get(0);
    File blockFile=DataNodeTestUtils.getBlockFile(dn,bpid,block);
    File metaFile=DataNodeTestUtils.getMetaFile(dn,bpid,block);
    assertTrue("Could not delete the block file from the RBW folder",blockFile.delete());
    assertTrue("Could not delete the block meta file from the RBW folder",metaFile.delete());
    out.close();
    int liveReplicas=0;
    while (true) {
      if ((liveReplicas=countReplicas(namesystem,blk).liveReplicas()) < 2) {
        LOG.info("Live Replicas after corruption: " + liveReplicas);
        break;
      }
      Thread.sleep(100);
    }
    assertEquals("There should be less than 2 replicas in the " + "liveReplicasMap",1,liveReplicas);
    while (true) {
      if ((liveReplicas=countReplicas(namesystem,blk).liveReplicas()) > 1) {
        LOG.info("Live Replicas after Rereplication: " + liveReplicas);
        break;
      }
      Thread.sleep(100);
    }
    assertEquals("There should be two live replicas",2,liveReplicas);
    while (true) {
      Thread.sleep(100);
      if (countReplicas(namesystem,blk).corruptReplicas() == 0) {
        LOG.info("Corrupt Replicas becomes 0");
        break;
      }
    }
  }
  finally {
    if (out != null) {
      out.close();
    }
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Regression test for HDFS-4799, a case where, upon restart, if there
 * were RWR replicas with out-of-date genstamps, the NN could accidentally
 * delete good replicas instead of the bad replicas.
 */
@Test(timeout=60000) public void testRWRInvalidation() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,RandomDeleterPolicy.class,BlockPlacementPolicy.class);
  conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,1);
  List testPaths=Lists.newArrayList();
  for (int i=0; i < 10; i++) {
    testPaths.add(new Path("/test" + i));
  }
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  try {
    List streams=Lists.newArrayList();
    try {
      for (      Path path : testPaths) {
        FSDataOutputStream out=cluster.getFileSystem().create(path,(short)2);
        streams.add(out);
        out.writeBytes("old gs data\n");
        out.hflush();
      }
      DataNodeProperties oldGenstampNode=cluster.stopDataNode(0);
      for (int i=0; i < streams.size(); i++) {
        Path path=testPaths.get(i);
        FSDataOutputStream out=streams.get(i);
        out.writeBytes("new gs data\n");
        out.hflush();
        cluster.getFileSystem().setReplication(path,(short)1);
        out.close();
      }
      LOG.info("=========================== restarting cluster");
      DataNodeProperties otherNode=cluster.stopDataNode(0);
      cluster.restartNameNode();
      cluster.restartDataNode(oldGenstampNode);
      cluster.waitActive();
      cluster.restartDataNode(otherNode);
      cluster.waitActive();
      cluster.getNameNode().getNamesystem().getBlockManager().computeInvalidateWork(2);
      cluster.triggerHeartbeats();
      HATestUtil.waitForDNDeletions(cluster);
      cluster.triggerDeletionReports();
      for (      Path path : testPaths) {
        String ret=DFSTestUtil.readFile(cluster.getFileSystem(),path);
        assertEquals("old gs data\n" + "new gs data\n",ret);
      }
    }
  finally {
      IOUtils.cleanup(LOG,streams.toArray(new Closeable[0]));
    }
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.blockmanagement.TestReplicationPolicy

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * In this testcase, client is is a node outside of file system.
 * So the 1st replica can be placed on any node. 
 * the 2nd replica should be placed on a different rack,
 * the 3rd replica should be placed on the same rack as the 2nd replica,
 * @throws Exception
 */
@Test public void testChooseTarget5() throws Exception {
  DatanodeDescriptor writerDesc=DFSTestUtil.getDatanodeDescriptor("7.7.7.7","/d2/r4");
  DatanodeStorageInfo[] targets;
  targets=chooseTarget(0,writerDesc);
  assertEquals(targets.length,0);
  targets=chooseTarget(1,writerDesc);
  assertEquals(targets.length,1);
  targets=chooseTarget(2,writerDesc);
  assertEquals(targets.length,2);
  assertFalse(isOnSameRack(targets[0],targets[1]));
  targets=chooseTarget(3,writerDesc);
  assertEquals(targets.length,3);
  assertTrue(isOnSameRack(targets[1],targets[2]));
  assertFalse(isOnSameRack(targets[0],targets[1]));
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testChooseTargetWithStaleNodes() throws Exception {
  dataNodes[0].setLastUpdate(Time.now() - staleInterval - 1);
  namenode.getNamesystem().getBlockManager().getDatanodeManager().getHeartbeatManager().heartbeatCheck();
  assertTrue(namenode.getNamesystem().getBlockManager().getDatanodeManager().shouldAvoidStaleDataNodesForWrite());
  DatanodeStorageInfo[] targets;
  targets=chooseTarget(1);
  assertEquals(targets.length,1);
  assertEquals(storages[1],targets[0]);
  Set excludedNodes=new HashSet();
  excludedNodes.add(dataNodes[1]);
  List chosenNodes=new ArrayList();
  targets=chooseTarget(1,chosenNodes,excludedNodes);
  assertEquals(targets.length,1);
  assertFalse(isOnSameRack(targets[0],dataNodes[0]));
  dataNodes[0].setLastUpdate(Time.now());
  namenode.getNamesystem().getBlockManager().getDatanodeManager().getHeartbeatManager().heartbeatCheck();
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

@Test public void testChooseTargetWithMoreThanHalfStaleNodes() throws Exception {
  HdfsConfiguration conf=new HdfsConfiguration();
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_WRITE_KEY,true);
  String[] hosts=new String[]{"host1","host2","host3","host4","host5","host6"};
  String[] racks=new String[]{"/d1/r1","/d1/r1","/d1/r2","/d1/r2","/d2/r3","/d2/r3"};
  MiniDFSCluster miniCluster=new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts).numDataNodes(hosts.length).build();
  miniCluster.waitActive();
  try {
    for (int i=0; i < 2; i++) {
      DataNode dn=miniCluster.getDataNodes().get(i);
      DataNodeTestUtils.setHeartbeatsDisabledForTests(dn,true);
      miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getDatanode(dn.getDatanodeId()).setLastUpdate(Time.now() - staleInterval - 1);
    }
    miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getHeartbeatManager().heartbeatCheck();
    int numStaleNodes=miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getNumStaleNodes();
    assertEquals(numStaleNodes,2);
    assertTrue(miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().shouldAvoidStaleDataNodesForWrite());
    DatanodeDescriptor staleNodeInfo=miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getDatanode(miniCluster.getDataNodes().get(0).getDatanodeId());
    BlockPlacementPolicy replicator=miniCluster.getNameNode().getNamesystem().getBlockManager().getBlockPlacementPolicy();
    DatanodeStorageInfo[] targets=replicator.chooseTarget(filename,3,staleNodeInfo,new ArrayList(),false,null,BLOCK_SIZE,StorageType.DEFAULT);
    assertEquals(targets.length,3);
    assertFalse(isOnSameRack(targets[0],staleNodeInfo));
    for (int i=0; i < 4; i++) {
      DataNode dn=miniCluster.getDataNodes().get(i);
      DataNodeTestUtils.setHeartbeatsDisabledForTests(dn,true);
      miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getDatanode(dn.getDatanodeId()).setLastUpdate(Time.now() - staleInterval - 1);
    }
    miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getHeartbeatManager().heartbeatCheck();
    numStaleNodes=miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getNumStaleNodes();
    assertEquals(numStaleNodes,4);
    assertFalse(miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().shouldAvoidStaleDataNodesForWrite());
    targets=replicator.chooseTarget(filename,3,staleNodeInfo,new ArrayList(),false,null,BLOCK_SIZE,StorageType.DEFAULT);
    assertEquals(targets.length,3);
    assertTrue(isOnSameRack(targets[0],staleNodeInfo));
    for (int i=2; i < 4; i++) {
      DataNode dn=miniCluster.getDataNodes().get(i);
      DataNodeTestUtils.setHeartbeatsDisabledForTests(dn,false);
      miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getDatanode(dn.getDatanodeId()).setLastUpdate(Time.now());
    }
    miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getHeartbeatManager().heartbeatCheck();
    numStaleNodes=miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getNumStaleNodes();
    assertEquals(numStaleNodes,2);
    assertTrue(miniCluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().shouldAvoidStaleDataNodesForWrite());
    targets=chooseTarget(3,staleNodeInfo);
    assertEquals(targets.length,3);
    assertFalse(isOnSameRack(targets[0],staleNodeInfo));
  }
  finally {
    miniCluster.shutdown();
  }
}

APIUtilityVerifier BooleanVerifier 

/** 
 * This testcase tests whether an IllegalArgumentException 
 * will be thrown when a value greater than 1 is retrieved by 
 * DFSUtil#getInvalidateWorkPctPerIteration
 */
@Test public void testGetInvalidateWorkPctPerIteration_GreaterThanOne(){
  Configuration conf=new Configuration();
  float blocksInvalidateWorkPct=DFSUtil.getInvalidateWorkPctPerIteration(conf);
  assertTrue(blocksInvalidateWorkPct > 0);
  conf.set(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,"1.5f");
  exception.expect(IllegalArgumentException.class);
  blocksInvalidateWorkPct=DFSUtil.getInvalidateWorkPctPerIteration(conf);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * This testcase tests whether the value returned by
 * DFSUtil.getReplWorkMultiplier() is positive,
 * and whether an IllegalArgumentException will be thrown 
 * when a non-positive value is retrieved
 */
@Test public void testGetReplWorkMultiplier(){
  Configuration conf=new Configuration();
  int blocksReplWorkMultiplier=DFSUtil.getReplWorkMultiplier(conf);
  assertTrue(blocksReplWorkMultiplier > 0);
  conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,"3");
  blocksReplWorkMultiplier=DFSUtil.getReplWorkMultiplier(conf);
  assertEquals(blocksReplWorkMultiplier,3);
  conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,"-1");
  exception.expect(IllegalArgumentException.class);
  blocksReplWorkMultiplier=DFSUtil.getReplWorkMultiplier(conf);
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * This testcase tests re-replication, when dataNodes[0] is already chosen.
 * So the 1st replica can be placed on random rack. 
 * the 2nd replica should be placed on different node by same rack as 
 * the 1st replica. The 3rd replica can be placed randomly.
 * @throws Exception
 */
@Test public void testRereplicate1() throws Exception {
  List chosenNodes=new ArrayList();
  chosenNodes.add(storages[0]);
  DatanodeStorageInfo[] targets;
  targets=chooseTarget(0,chosenNodes);
  assertEquals(targets.length,0);
  targets=chooseTarget(1,chosenNodes);
  assertEquals(targets.length,1);
  assertFalse(isOnSameRack(targets[0],dataNodes[0]));
  targets=chooseTarget(2,chosenNodes);
  assertEquals(targets.length,2);
  assertTrue(isOnSameRack(targets[0],dataNodes[0]));
  assertFalse(isOnSameRack(targets[0],targets[1]));
  targets=chooseTarget(3,chosenNodes);
  assertEquals(targets.length,3);
  assertTrue(isOnSameRack(targets[0],dataNodes[0]));
  assertFalse(isOnSameRack(targets[0],targets[2]));
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * In this testcase, client is dataNodes[0], but the dataNodes[1] is
 * not allowed to be chosen. So the 1st replica should be
 * placed on dataNodes[0], the 2nd replica should be placed on a different
 * rack, the 3rd should be on same rack as the 2nd replica, and the rest
 * should be placed on a third rack.
 * @throws Exception
 */
@Test public void testChooseTarget2() throws Exception {
  Set excludedNodes;
  DatanodeStorageInfo[] targets;
  List chosenNodes=new ArrayList();
  excludedNodes=new HashSet();
  excludedNodes.add(dataNodes[1]);
  targets=chooseTarget(0,chosenNodes,excludedNodes);
  assertEquals(targets.length,0);
  excludedNodes.clear();
  chosenNodes.clear();
  excludedNodes.add(dataNodes[1]);
  targets=chooseTarget(1,chosenNodes,excludedNodes);
  assertEquals(targets.length,1);
  assertEquals(storages[0],targets[0]);
  excludedNodes.clear();
  chosenNodes.clear();
  excludedNodes.add(dataNodes[1]);
  targets=chooseTarget(2,chosenNodes,excludedNodes);
  assertEquals(targets.length,2);
  assertEquals(storages[0],targets[0]);
  assertFalse(isOnSameRack(targets[0],targets[1]));
  excludedNodes.clear();
  chosenNodes.clear();
  excludedNodes.add(dataNodes[1]);
  targets=chooseTarget(3,chosenNodes,excludedNodes);
  assertEquals(targets.length,3);
  assertEquals(storages[0],targets[0]);
  assertFalse(isOnSameRack(targets[0],targets[1]));
  assertTrue(isOnSameRack(targets[1],targets[2]));
  excludedNodes.clear();
  chosenNodes.clear();
  excludedNodes.add(dataNodes[1]);
  targets=chooseTarget(4,chosenNodes,excludedNodes);
  assertEquals(targets.length,4);
  assertEquals(storages[0],targets[0]);
  for (int i=1; i < 4; i++) {
    assertFalse(isOnSameRack(targets[0],targets[i]));
  }
  assertTrue(isOnSameRack(targets[1],targets[2]) || isOnSameRack(targets[2],targets[3]));
  assertFalse(isOnSameRack(targets[1],targets[3]));
  excludedNodes.clear();
  chosenNodes.clear();
  excludedNodes.add(dataNodes[1]);
  chosenNodes.add(storages[2]);
  targets=replicator.chooseTarget(filename,1,dataNodes[0],chosenNodes,true,excludedNodes,BLOCK_SIZE,StorageType.DEFAULT);
  System.out.println("targets=" + Arrays.asList(targets));
  assertEquals(2,targets.length);
  int i=0;
  for (; i < targets.length && !storages[2].equals(targets[i]); i++)   ;
  assertTrue(i < targets.length);
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * This testcase tests re-replication, 
 * when dataNodes[0] and dataNodes[2] are already chosen.
 * So the 1st replica should be placed on the rack that the writer resides. 
 * the rest replicas can be placed randomly,
 * @throws Exception
 */
@Test public void testRereplicate3() throws Exception {
  List chosenNodes=new ArrayList();
  chosenNodes.add(storages[0]);
  chosenNodes.add(storages[2]);
  DatanodeStorageInfo[] targets;
  targets=chooseTarget(0,chosenNodes);
  assertEquals(targets.length,0);
  targets=chooseTarget(1,chosenNodes);
  assertEquals(targets.length,1);
  assertTrue(isOnSameRack(targets[0],dataNodes[0]));
  assertFalse(isOnSameRack(targets[0],dataNodes[2]));
  targets=chooseTarget(1,dataNodes[2],chosenNodes);
  assertEquals(targets.length,1);
  assertTrue(isOnSameRack(targets[0],dataNodes[2]));
  assertFalse(isOnSameRack(targets[0],dataNodes[0]));
  targets=chooseTarget(2,chosenNodes);
  assertEquals(targets.length,2);
  assertTrue(isOnSameRack(targets[0],dataNodes[0]));
  targets=chooseTarget(2,dataNodes[2],chosenNodes);
  assertEquals(targets.length,2);
  assertTrue(isOnSameRack(targets[0],dataNodes[2]));
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * In this testcase, it tries to choose more targets than available nodes and
 * check the result. 
 * @throws Exception
 */
@Test public void testChooseTargetWithMoreThanAvailableNodes() throws Exception {
  for (int i=0; i < 2; i++) {
    updateHeartbeatWithUsage(dataNodes[i],2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,0L,(HdfsConstants.MIN_BLOCKS_FOR_WRITE - 1) * BLOCK_SIZE,0L,0L,0L,0,0);
  }
  final LogVerificationAppender appender=new LogVerificationAppender();
  final Logger logger=Logger.getRootLogger();
  logger.addAppender(appender);
  DatanodeStorageInfo[] targets=chooseTarget(NUM_OF_DATANODES);
  assertEquals(targets.length,NUM_OF_DATANODES - 2);
  final List log=appender.getLog();
  assertNotNull(log);
  assertFalse(log.size() == 0);
  final LoggingEvent lastLogEntry=log.get(log.size() - 1);
  assertTrue(Level.WARN.isGreaterOrEqual(lastLogEntry.getLevel()));
  assertTrue(((String)lastLogEntry.getMessage()).contains("in need of 2"));
  for (int i=0; i < 2; i++) {
    updateHeartbeatWithUsage(dataNodes[i],2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,0L,HdfsConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,0L,0L,0L,0,0);
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * This testcase tests whether the default value returned by
 * DFSUtil.getInvalidateWorkPctPerIteration() is positive, 
 * and whether an IllegalArgumentException will be thrown 
 * when 0.0f is retrieved
 */
@Test public void testGetInvalidateWorkPctPerIteration(){
  Configuration conf=new Configuration();
  float blocksInvalidateWorkPct=DFSUtil.getInvalidateWorkPctPerIteration(conf);
  assertTrue(blocksInvalidateWorkPct > 0);
  conf.set(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,"0.5f");
  blocksInvalidateWorkPct=DFSUtil.getInvalidateWorkPctPerIteration(conf);
  assertEquals(blocksInvalidateWorkPct,0.5f,blocksInvalidateWorkPct * 1e-7);
  conf.set(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,"1.0f");
  blocksInvalidateWorkPct=DFSUtil.getInvalidateWorkPctPerIteration(conf);
  assertEquals(blocksInvalidateWorkPct,1.0f,blocksInvalidateWorkPct * 1e-7);
  conf.set(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,"0.0f");
  exception.expect(IllegalArgumentException.class);
  blocksInvalidateWorkPct=DFSUtil.getInvalidateWorkPctPerIteration(conf);
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * This testcase tests re-replication, 
 * when dataNodes[0] and dataNodes[1] are already chosen.
 * So the 1st replica should be placed on a different rack than rack 1. 
 * the rest replicas can be placed randomly,
 * @throws Exception
 */
@Test public void testRereplicate2() throws Exception {
  List chosenNodes=new ArrayList();
  chosenNodes.add(storages[0]);
  chosenNodes.add(storages[1]);
  DatanodeStorageInfo[] targets;
  targets=chooseTarget(0,chosenNodes);
  assertEquals(targets.length,0);
  targets=chooseTarget(1,chosenNodes);
  assertEquals(targets.length,1);
  assertFalse(isOnSameRack(targets[0],dataNodes[0]));
  targets=chooseTarget(2,chosenNodes);
  assertEquals(targets.length,2);
  assertFalse(isOnSameRack(targets[0],dataNodes[0]));
  assertFalse(isOnSameRack(targets[1],dataNodes[0]));
}

APIUtilityVerifier BooleanVerifier 

/** 
 * This testcase tests whether an IllegalArgumentException 
 * will be thrown when a negative value is retrieved by 
 * DFSUtil#getInvalidateWorkPctPerIteration
 */
@Test public void testGetInvalidateWorkPctPerIteration_NegativeValue(){
  Configuration conf=new Configuration();
  float blocksInvalidateWorkPct=DFSUtil.getInvalidateWorkPctPerIteration(conf);
  assertTrue(blocksInvalidateWorkPct > 0);
  conf.set(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,"-0.5f");
  exception.expect(IllegalArgumentException.class);
  blocksInvalidateWorkPct=DFSUtil.getInvalidateWorkPctPerIteration(conf);
}

Class: org.apache.hadoop.hdfs.server.blockmanagement.TestReplicationPolicyConsiderLoad

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests that chooseTarget with considerLoad set to true correctly calculates
 * load with decommissioned nodes.
 */
@Test public void testChooseTargetWithDecomNodes() throws IOException {
  namenode.getNamesystem().writeLock();
  try {
    String blockPoolId=namenode.getNamesystem().getBlockPoolId();
    dnManager.handleHeartbeat(dnrList.get(3),BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[3]),blockPoolId,dataNodes[3].getCacheCapacity(),dataNodes[3].getCacheRemaining(),2,0,0);
    dnManager.handleHeartbeat(dnrList.get(4),BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[4]),blockPoolId,dataNodes[4].getCacheCapacity(),dataNodes[4].getCacheRemaining(),4,0,0);
    dnManager.handleHeartbeat(dnrList.get(5),BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[5]),blockPoolId,dataNodes[5].getCacheCapacity(),dataNodes[5].getCacheRemaining(),4,0,0);
    final int load=2 + 4 + 4;
    FSNamesystem fsn=namenode.getNamesystem();
    assertEquals((double)load / 6,fsn.getInServiceXceiverAverage(),EPSILON);
    for (int i=0; i < 3; i++) {
      DatanodeDescriptor d=dnManager.getDatanode(dnrList.get(i));
      dnManager.startDecommission(d);
      d.setDecommissioned();
    }
    assertEquals((double)load / 3,fsn.getInServiceXceiverAverage(),EPSILON);
    DatanodeStorageInfo[] targets=namenode.getNamesystem().getBlockManager().getBlockPlacementPolicy().chooseTarget("testFile.txt",3,dataNodes[0],new ArrayList(),false,null,1024,StorageType.DEFAULT);
    assertEquals(3,targets.length);
    Set targetSet=new HashSet(Arrays.asList(targets));
    for (int i=3; i < storages.length; i++) {
      assertTrue(targetSet.contains(storages[i]));
    }
  }
  finally {
    dataNodes[0].stopDecommission();
    dataNodes[1].stopDecommission();
    dataNodes[2].stopDecommission();
    namenode.getNamesystem().writeUnlock();
  }
  NameNode.LOG.info("Done working on it");
}

Class: org.apache.hadoop.hdfs.server.blockmanagement.TestReplicationPolicyWithNodeGroup

APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testChooseTargetWithDependencies() throws Exception {
  for (int i=0; i < NUM_OF_DATANODES; i++) {
    cluster.remove(dataNodes[i]);
  }
  for (int i=0; i < NUM_OF_DATANODES_MORE_TARGETS; i++) {
    DatanodeDescriptor node=dataNodesInMoreTargetsCase[i];
    if (cluster.contains(node)) {
      cluster.remove(node);
    }
  }
  Host2NodesMap host2DatanodeMap=namenode.getNamesystem().getBlockManager().getDatanodeManager().getHost2DatanodeMap();
  for (int i=0; i < NUM_OF_DATANODES_FOR_DEPENDENCIES; i++) {
    cluster.add(dataNodesForDependencies[i]);
    host2DatanodeMap.add(dataNodesForDependencies[i]);
  }
  dataNodesForDependencies[1].addDependentHostName(dataNodesForDependencies[2].getHostName());
  dataNodesForDependencies[2].addDependentHostName(dataNodesForDependencies[1].getHostName());
  dataNodesForDependencies[3].addDependentHostName(dataNodesForDependencies[4].getHostName());
  dataNodesForDependencies[4].addDependentHostName(dataNodesForDependencies[3].getHostName());
  for (int i=0; i < NUM_OF_DATANODES_FOR_DEPENDENCIES; i++) {
    updateHeartbeatWithUsage(dataNodesForDependencies[i],2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,0L,2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,0L,0L,0L,0,0);
  }
  List chosenNodes=new ArrayList();
  DatanodeStorageInfo[] targets;
  Set excludedNodes=new HashSet();
  excludedNodes.add(dataNodesForDependencies[5]);
  targets=chooseTarget(3,dataNodesForDependencies[1],chosenNodes,excludedNodes);
  assertEquals(targets.length,2);
  assertEquals(targets[0],storagesForDependencies[1]);
  assertTrue(targets[1].equals(storagesForDependencies[3]) || targets[1].equals(storagesForDependencies[4]));
  assertEquals(excludedNodes.size(),NUM_OF_DATANODES_FOR_DEPENDENCIES);
  for (int i=0; i < NUM_OF_DATANODES_FOR_DEPENDENCIES; i++) {
    assertTrue(excludedNodes.contains(dataNodesForDependencies[i]));
  }
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * This testcase tests re-replication, 
 * when dataNodes[0] and dataNodes[3] are already chosen.
 * So the 1st replica should be placed on the rack that the writer resides. 
 * the rest replicas can be placed randomly,
 * @throws Exception
 */
@Test public void testRereplicate3() throws Exception {
  setupDataNodeCapacity();
  List chosenNodes=new ArrayList();
  chosenNodes.add(storages[0]);
  chosenNodes.add(storages[3]);
  DatanodeStorageInfo[] targets;
  targets=chooseTarget(0,chosenNodes);
  assertEquals(targets.length,0);
  targets=chooseTarget(1,chosenNodes);
  assertEquals(targets.length,1);
  assertTrue(isOnSameRack(dataNodes[0],targets[0]));
  assertFalse(isOnSameRack(dataNodes[3],targets[0]));
  targets=chooseTarget(1,dataNodes[3],chosenNodes);
  assertEquals(targets.length,1);
  assertTrue(isOnSameRack(dataNodes[3],targets[0]));
  assertFalse(isOnSameNodeGroup(dataNodes[3],targets[0]));
  assertFalse(isOnSameRack(dataNodes[0],targets[0]));
  targets=chooseTarget(2,chosenNodes);
  assertEquals(targets.length,2);
  assertTrue(isOnSameRack(dataNodes[0],targets[0]));
  assertFalse(isOnSameNodeGroup(dataNodes[0],targets[0]));
  targets=chooseTarget(2,dataNodes[3],chosenNodes);
  assertEquals(targets.length,2);
  assertTrue(isOnSameRack(dataNodes[3],targets[0]));
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * This testcase tests re-replication, 
 * when dataNodes[0] and dataNodes[1] are already chosen.
 * So the 1st replica should be placed on a different rack of rack 1. 
 * the rest replicas can be placed randomly,
 * @throws Exception
 */
@Test public void testRereplicate2() throws Exception {
  setupDataNodeCapacity();
  List chosenNodes=new ArrayList();
  chosenNodes.add(storages[0]);
  chosenNodes.add(storages[1]);
  DatanodeStorageInfo[] targets;
  targets=chooseTarget(0,chosenNodes);
  assertEquals(targets.length,0);
  targets=chooseTarget(1,chosenNodes);
  assertEquals(targets.length,1);
  assertFalse(isOnSameRack(dataNodes[0],targets[0]));
  targets=chooseTarget(2,chosenNodes);
  assertEquals(targets.length,2);
  assertFalse(isOnSameRack(dataNodes[0],targets[0]) && isOnSameRack(dataNodes[0],targets[1]));
}

APIUtilityVerifier BooleanVerifier 

/** 
 * Test re-replication policy in boundary case.
 * Rack 2 has only one node group & the node in this node group is chosen
 * Rack 1 has two nodegroups & one of them is chosen.
 * Replica policy should choose the node from node group of Rack1 but not the
 * same nodegroup with chosen nodes.
 */
@Test public void testRereplicateOnBoundaryTopology() throws Exception {
  for (int i=0; i < NUM_OF_DATANODES_BOUNDARY; i++) {
    updateHeartbeatWithUsage(dataNodesInBoundaryCase[i],2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,0L,2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE,0L,0L,0L,0,0);
  }
  List chosenNodes=new ArrayList();
  chosenNodes.add(storagesInBoundaryCase[0]);
  chosenNodes.add(storagesInBoundaryCase[5]);
  DatanodeStorageInfo[] targets;
  targets=chooseTarget(1,dataNodesInBoundaryCase[0],chosenNodes);
  assertFalse(isOnSameNodeGroup(dataNodesInBoundaryCase[0],targets[0]));
  assertFalse(isOnSameNodeGroup(dataNodesInBoundaryCase[5],targets[0]));
  assertTrue(checkTargetsOnDifferentNodeGroup(targets));
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * This testcase tests re-replication, when dataNodes[0] is already chosen.
 * So the 1st replica can be placed on random rack. 
 * the 2nd replica should be placed on different node and nodegroup by same rack as 
 * the 1st replica. The 3rd replica can be placed randomly.
 * @throws Exception
 */
@Test public void testRereplicate1() throws Exception {
  setupDataNodeCapacity();
  List chosenNodes=new ArrayList();
  chosenNodes.add(storages[0]);
  DatanodeStorageInfo[] targets;
  targets=chooseTarget(0,chosenNodes);
  assertEquals(targets.length,0);
  targets=chooseTarget(1,chosenNodes);
  assertEquals(targets.length,1);
  assertFalse(isOnSameRack(dataNodes[0],targets[0]));
  targets=chooseTarget(2,chosenNodes);
  assertEquals(targets.length,2);
  assertTrue(isOnSameRack(dataNodes[0],targets[0]));
  assertFalse(isOnSameRack(targets[0],targets[1]));
  targets=chooseTarget(3,chosenNodes);
  assertEquals(targets.length,3);
  assertTrue(isOnSameRack(dataNodes[0],targets[0]));
  assertFalse(isOnSameNodeGroup(dataNodes[0],targets[0]));
  assertFalse(isOnSameRack(targets[0],targets[2]));
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * In this testcase, client is is a node outside of file system.
 * So the 1st replica can be placed on any node. 
 * the 2nd replica should be placed on a different rack,
 * the 3rd replica should be placed on the same rack as the 2nd replica,
 * @throws Exception
 */
@Test public void testChooseTarget5() throws Exception {
  setupDataNodeCapacity();
  DatanodeStorageInfo[] targets;
  targets=chooseTarget(0,NODE);
  assertEquals(targets.length,0);
  targets=chooseTarget(1,NODE);
  assertEquals(targets.length,1);
  targets=chooseTarget(2,NODE);
  assertEquals(targets.length,2);
  assertFalse(isOnSameRack(targets[0],targets[1]));
  targets=chooseTarget(3,NODE);
  assertEquals(targets.length,3);
  assertTrue(isOnSameRack(targets[1],targets[2]));
  assertFalse(isOnSameRack(targets[0],targets[1]));
  verifyNoTwoTargetsOnSameNodeGroup(targets);
}

Class: org.apache.hadoop.hdfs.server.common.TestGetUriFromString

APIUtilityVerifier NullVerifier 

/** 
 * Test for a relative path, os independent
 * @throws IOException 
 */
@Test public void testRelativePathAsURI() throws IOException {
  URI u=Util.stringAsURI(RELATIVE_FILE_PATH);
  LOG.info("Uri: " + u);
  assertNotNull(u);
}

APIUtilityVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test for an OS dependent absolute paths.
 * @throws IOException 
 */
@Test public void testAbsolutePathAsURI() throws IOException {
  URI u=null;
  u=Util.stringAsURI(ABSOLUTE_PATH_WINDOWS);
  assertNotNull("Uri should not be null for Windows path" + ABSOLUTE_PATH_WINDOWS,u);
  assertEquals(URI_FILE_SCHEMA,u.getScheme());
  u=Util.stringAsURI(ABSOLUTE_PATH_UNIX);
  assertNotNull("Uri should not be null for Unix path" + ABSOLUTE_PATH_UNIX,u);
  assertEquals(URI_FILE_SCHEMA,u.getScheme());
}

APIUtilityVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test for a URI
 * @throws IOException 
 */
@Test public void testURI() throws IOException {
  LOG.info("Testing correct Unix URI: " + URI_UNIX);
  URI u=Util.stringAsURI(URI_UNIX);
  LOG.info("Uri: " + u);
  assertNotNull("Uri should not be null at this point",u);
  assertEquals(URI_FILE_SCHEMA,u.getScheme());
  assertEquals(URI_PATH_UNIX,u.getPath());
  LOG.info("Testing correct windows URI: " + URI_WINDOWS);
  u=Util.stringAsURI(URI_WINDOWS);
  LOG.info("Uri: " + u);
  assertNotNull("Uri should not be null at this point",u);
  assertEquals(URI_FILE_SCHEMA,u.getScheme());
  assertEquals(URI_PATH_WINDOWS.replace("%20"," "),u.getPath());
}

Class: org.apache.hadoop.hdfs.server.common.TestJspHelper

APIUtilityVerifier UtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testGetUgiFromToken() throws IOException {
  conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY,"hdfs://localhost:4321/");
  ServletContext context=mock(ServletContext.class);
  String realUser="TheDoctor";
  String user="TheNurse";
  conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION,"kerberos");
  UserGroupInformation.setConfiguration(conf);
  UserGroupInformation ugi;
  HttpServletRequest request;
  Text ownerText=new Text(user);
  DelegationTokenIdentifier dtId=new DelegationTokenIdentifier(ownerText,ownerText,new Text(realUser));
  Token token=new Token(dtId,new DummySecretManager(0,0,0,0));
  String tokenString=token.encodeToUrlString();
  request=getMockRequest(null,null,null);
  when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(tokenString);
  ugi=JspHelper.getUGI(context,request,conf);
  Assert.assertNotNull(ugi.getRealUser());
  Assert.assertEquals(ugi.getRealUser().getShortUserName(),realUser);
  Assert.assertEquals(ugi.getShortUserName(),user);
  checkUgiFromToken(ugi);
  request=getMockRequest(realUser,null,null);
  when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(tokenString);
  ugi=JspHelper.getUGI(context,request,conf);
  Assert.assertNotNull(ugi.getRealUser());
  Assert.assertEquals(ugi.getRealUser().getShortUserName(),realUser);
  Assert.assertEquals(ugi.getShortUserName(),user);
  checkUgiFromToken(ugi);
  request=getMockRequest("rogue",null,null);
  when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(tokenString);
  ugi=JspHelper.getUGI(context,request,conf);
  Assert.assertNotNull(ugi.getRealUser());
  Assert.assertEquals(ugi.getRealUser().getShortUserName(),realUser);
  Assert.assertEquals(ugi.getShortUserName(),user);
  checkUgiFromToken(ugi);
  request=getMockRequest(null,user,null);
  when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(tokenString);
  ugi=JspHelper.getUGI(context,request,conf);
  Assert.assertNotNull(ugi.getRealUser());
  Assert.assertEquals(ugi.getRealUser().getShortUserName(),realUser);
  Assert.assertEquals(ugi.getShortUserName(),user);
  checkUgiFromToken(ugi);
  request=getMockRequest(null,null,"rogue");
  when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(tokenString);
  try {
    JspHelper.getUGI(context,request,conf);
    Assert.fail("bad request allowed");
  }
 catch (  IOException ioe) {
    Assert.assertEquals("Usernames not matched: name=rogue != expected=" + user,ioe.getMessage());
  }
  request=getMockRequest(null,user,"rogue");
  when(request.getParameter(JspHelper.DELEGATION_PARAMETER_NAME)).thenReturn(tokenString);
  try {
    JspHelper.getUGI(context,request,conf);
    Assert.fail("bad request allowed");
  }
 catch (  IOException ioe) {
    Assert.assertEquals("Usernames not matched: name=rogue != expected=" + user,ioe.getMessage());
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testGetNonProxyUgi() throws IOException {
  conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY,"hdfs://localhost:4321/");
  ServletContext context=mock(ServletContext.class);
  String realUser="TheDoctor";
  String user="TheNurse";
  conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION,"kerberos");
  UserGroupInformation.setConfiguration(conf);
  UserGroupInformation ugi;
  HttpServletRequest request;
  request=getMockRequest(null,null,null);
  try {
    JspHelper.getUGI(context,request,conf);
    Assert.fail("bad request allowed");
  }
 catch (  IOException ioe) {
    Assert.assertEquals("Security enabled but user not authenticated by filter",ioe.getMessage());
  }
  request=getMockRequest(null,realUser,null);
  try {
    JspHelper.getUGI(context,request,conf);
    Assert.fail("bad request allowed");
  }
 catch (  IOException ioe) {
    Assert.assertEquals("Security enabled but user not authenticated by filter",ioe.getMessage());
  }
  request=getMockRequest(realUser,null,null);
  ugi=JspHelper.getUGI(context,request,conf);
  Assert.assertNull(ugi.getRealUser());
  Assert.assertEquals(ugi.getShortUserName(),realUser);
  checkUgiFromAuth(ugi);
  request=getMockRequest(realUser,realUser,null);
  ugi=JspHelper.getUGI(context,request,conf);
  Assert.assertNull(ugi.getRealUser());
  Assert.assertEquals(ugi.getShortUserName(),realUser);
  checkUgiFromAuth(ugi);
  request=getMockRequest(realUser,user,null);
  try {
    JspHelper.getUGI(context,request,conf);
    Assert.fail("bad request allowed");
  }
 catch (  IOException ioe) {
    Assert.assertEquals("Usernames not matched: name=" + user + " != expected="+ realUser,ioe.getMessage());
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

@Test public void testReadWriteReplicaState(){
  try {
    DataOutputBuffer out=new DataOutputBuffer();
    DataInputBuffer in=new DataInputBuffer();
    for (    HdfsServerConstants.ReplicaState repState : HdfsServerConstants.ReplicaState.values()) {
      repState.write(out);
      in.reset(out.getData(),out.getLength());
      HdfsServerConstants.ReplicaState result=HdfsServerConstants.ReplicaState.read(in);
      assertTrue("testReadWrite error !!!",repState == result);
      out.reset();
      in.reset();
    }
  }
 catch (  Exception ex) {
    fail("testReadWrite ex error ReplicaState");
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testGetProxyUgi() throws IOException {
  conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY,"hdfs://localhost:4321/");
  ServletContext context=mock(ServletContext.class);
  String realUser="TheDoctor";
  String user="TheNurse";
  conf.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION,"kerberos");
  conf.set(DefaultImpersonationProvider.getTestProvider().getProxySuperuserGroupConfKey(realUser),"*");
  conf.set(DefaultImpersonationProvider.getTestProvider().getProxySuperuserIpConfKey(realUser),"*");
  ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
  UserGroupInformation.setConfiguration(conf);
  UserGroupInformation ugi;
  HttpServletRequest request;
  request=getMockRequest(null,null,user);
  try {
    JspHelper.getUGI(context,request,conf);
    Assert.fail("bad request allowed");
  }
 catch (  IOException ioe) {
    Assert.assertEquals("Security enabled but user not authenticated by filter",ioe.getMessage());
  }
  request=getMockRequest(null,realUser,user);
  try {
    JspHelper.getUGI(context,request,conf);
    Assert.fail("bad request allowed");
  }
 catch (  IOException ioe) {
    Assert.assertEquals("Security enabled but user not authenticated by filter",ioe.getMessage());
  }
  request=getMockRequest(realUser,null,user);
  ugi=JspHelper.getUGI(context,request,conf);
  Assert.assertNotNull(ugi.getRealUser());
  Assert.assertEquals(ugi.getRealUser().getShortUserName(),realUser);
  Assert.assertEquals(ugi.getShortUserName(),user);
  checkUgiFromAuth(ugi);
  request=getMockRequest(realUser,realUser,user);
  ugi=JspHelper.getUGI(context,request,conf);
  Assert.assertNotNull(ugi.getRealUser());
  Assert.assertEquals(ugi.getRealUser().getShortUserName(),realUser);
  Assert.assertEquals(ugi.getShortUserName(),user);
  checkUgiFromAuth(ugi);
  request=getMockRequest(realUser,user,user);
  try {
    JspHelper.getUGI(context,request,conf);
    Assert.fail("bad request allowed");
  }
 catch (  IOException ioe) {
    Assert.assertEquals("Usernames not matched: name=" + user + " != expected="+ realUser,ioe.getMessage());
  }
  try {
    request=getMockRequest(user,null,realUser);
    JspHelper.getUGI(context,request,conf);
    Assert.fail("bad proxy request allowed");
  }
 catch (  AuthorizationException ae) {
    Assert.assertEquals("User: " + user + " is not allowed to impersonate "+ realUser,ae.getMessage());
  }
  try {
    request=getMockRequest(user,user,realUser);
    JspHelper.getUGI(context,request,conf);
    Assert.fail("bad proxy request allowed");
  }
 catch (  AuthorizationException ae) {
    Assert.assertEquals("User: " + user + " is not allowed to impersonate "+ realUser,ae.getMessage());
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.BlockReportTestBase

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * The test set the configuration parameters for a large block size and
 * restarts initiated single-node cluster.
 * Then it writes a file > block_size and closes it.
 * The second datanode is started in the cluster.
 * As soon as the replication process is started and at least one TEMPORARY
 * replica is found test forces BlockReport process and checks
 * if the TEMPORARY replica isn't reported on it.
 * Eventually, the configuration is being restored into the original state.
 * @throws IOException in case of an error
 */
@Test(timeout=300000) public void blockReport_08() throws IOException {
  final String METHOD_NAME=GenericTestUtils.getMethodName();
  Path filePath=new Path("/" + METHOD_NAME + ".dat");
  final int DN_N1=DN_N0 + 1;
  final int bytesChkSum=1024 * 1000;
  conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY,bytesChkSum);
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,6 * bytesChkSum);
  shutDownCluster();
  startUpCluster();
  try {
    ArrayList blocks=writeFile(METHOD_NAME,12 * bytesChkSum,filePath);
    Block bl=findBlock(filePath,12 * bytesChkSum);
    BlockChecker bc=new BlockChecker(filePath);
    bc.start();
    waitForTempReplica(bl,DN_N1);
    DataNode dn=cluster.getDataNodes().get(DN_N1);
    String poolId=cluster.getNamesystem().getBlockPoolId();
    DatanodeRegistration dnR=dn.getDNRegistrationForBP(poolId);
    StorageBlockReport[] reports=getBlockReports(dn,poolId,false,false);
    sendBlockReports(dnR,poolId,reports);
    printStats();
    assertEquals("Wrong number of PendingReplication blocks",blocks.size(),cluster.getNamesystem().getPendingReplicationBlocks());
    try {
      bc.join();
    }
 catch (    InterruptedException e) {
    }
  }
  finally {
    resetConfiguration();
  }
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test write a file, verifies and closes it. Then the length of the blocks
 * are messed up and BlockReport is forced.
 * The modification of blocks' length has to be ignored
 * @throws java.io.IOException on an error
 */
@Test(timeout=300000) public void blockReport_01() throws IOException {
  final String METHOD_NAME=GenericTestUtils.getMethodName();
  Path filePath=new Path("/" + METHOD_NAME + ".dat");
  ArrayList blocks=prepareForRide(filePath,METHOD_NAME,FILE_SIZE);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Number of blocks allocated " + blocks.size());
  }
  long[] oldLengths=new long[blocks.size()];
  int tempLen;
  for (int i=0; i < blocks.size(); i++) {
    Block b=blocks.get(i);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Block " + b.getBlockName() + " before\t"+ "Size "+ b.getNumBytes());
    }
    oldLengths[i]=b.getNumBytes();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Setting new length");
    }
    tempLen=rand.nextInt(BLOCK_SIZE);
    b.set(b.getBlockId(),tempLen,b.getGenerationStamp());
    if (LOG.isDebugEnabled()) {
      LOG.debug("Block " + b.getBlockName() + " after\t "+ "Size "+ b.getNumBytes());
    }
  }
  DataNode dn=cluster.getDataNodes().get(DN_N0);
  String poolId=cluster.getNamesystem().getBlockPoolId();
  DatanodeRegistration dnR=dn.getDNRegistrationForBP(poolId);
  StorageBlockReport[] reports=getBlockReports(dn,poolId,false,false);
  sendBlockReports(dnR,poolId,reports);
  List blocksAfterReport=DFSTestUtil.getAllBlocks(fs.open(filePath));
  if (LOG.isDebugEnabled()) {
    LOG.debug("After mods: Number of blocks allocated " + blocksAfterReport.size());
  }
  for (int i=0; i < blocksAfterReport.size(); i++) {
    ExtendedBlock b=blocksAfterReport.get(i).getBlock();
    assertEquals("Length of " + i + "th block is incorrect",oldLengths[i],b.getNumBytes());
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.TestBPOfferService

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test datanode block pool initialization error handling.
 * Failure in initializing a block pool should not cause NPE.
 */
@Test public void testBPInitErrorHandling() throws Exception {
  final DataNode mockDn=Mockito.mock(DataNode.class);
  Mockito.doReturn(true).when(mockDn).shouldRun();
  Configuration conf=new Configuration();
  File dnDataDir=new File(new File(TEST_BUILD_DATA,"testBPInitErrorHandling"),"data");
  conf.set(DFS_DATANODE_DATA_DIR_KEY,dnDataDir.toURI().toString());
  Mockito.doReturn(conf).when(mockDn).getConf();
  Mockito.doReturn(new DNConf(conf)).when(mockDn).getDnConf();
  Mockito.doReturn(DataNodeMetrics.create(conf,"fake dn")).when(mockDn).getMetrics();
  final AtomicInteger count=new AtomicInteger();
  Mockito.doAnswer(new Answer(){
    @Override public Void answer(    InvocationOnMock invocation) throws Throwable {
      if (count.getAndIncrement() == 0) {
        throw new IOException("faked initBlockPool exception");
      }
      Mockito.doReturn(mockFSDataset).when(mockDn).getFSDataset();
      return null;
    }
  }
).when(mockDn).initBlockPool(Mockito.any(BPOfferService.class));
  BPOfferService bpos=setupBPOSForNNs(mockDn,mockNN1,mockNN2);
  List actors=bpos.getBPServiceActors();
  assertEquals(2,actors.size());
  bpos.start();
  try {
    waitForInitialization(bpos);
    waitForBlockReport(mockNN1,mockNN2);
  }
  finally {
    bpos.stop();
  }
}

APIUtilityVerifier InternalCallVerifier IdentityVerifier NullVerifier HybridVerifier 

/** 
 * Test that the DataNode determines the active NameNode correctly
 * based on the HA-related information in heartbeat responses.
 * See HDFS-2627.
 */
@Test public void testPickActiveNameNode() throws Exception {
  BPOfferService bpos=setupBPOSForNNs(mockNN1,mockNN2);
  bpos.start();
  try {
    waitForInitialization(bpos);
    assertNull(bpos.getActiveNN());
    mockHaStatuses[0]=new NNHAStatusHeartbeat(HAServiceState.ACTIVE,1);
    bpos.triggerHeartbeatForTests();
    assertSame(mockNN1,bpos.getActiveNN());
    mockHaStatuses[1]=new NNHAStatusHeartbeat(HAServiceState.ACTIVE,2);
    bpos.triggerHeartbeatForTests();
    assertSame(mockNN2,bpos.getActiveNN());
    bpos.triggerHeartbeatForTests();
    assertSame(mockNN2,bpos.getActiveNN());
    mockHaStatuses[1]=new NNHAStatusHeartbeat(HAServiceState.STANDBY,2);
    bpos.triggerHeartbeatForTests();
    assertNull(bpos.getActiveNN());
    mockHaStatuses[0]=new NNHAStatusHeartbeat(HAServiceState.ACTIVE,3);
    bpos.triggerHeartbeatForTests();
    assertSame(mockNN1,bpos.getActiveNN());
  }
  finally {
    bpos.stop();
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test that the BPOS can register to talk to two different NNs,
 * sends block reports to both, etc.
 */
@Test public void testBasicFunctionality() throws Exception {
  BPOfferService bpos=setupBPOSForNNs(mockNN1,mockNN2);
  bpos.start();
  try {
    waitForInitialization(bpos);
    Mockito.verify(mockNN1).registerDatanode(Mockito.any(DatanodeRegistration.class));
    Mockito.verify(mockNN2).registerDatanode(Mockito.any(DatanodeRegistration.class));
    waitForBlockReport(mockNN1);
    waitForBlockReport(mockNN2);
    bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK,"","");
    ReceivedDeletedBlockInfo[] ret=waitForBlockReceived(FAKE_BLOCK,mockNN1);
    assertEquals(1,ret.length);
    assertEquals(FAKE_BLOCK.getLocalBlock(),ret[0].getBlock());
    ret=waitForBlockReceived(FAKE_BLOCK,mockNN2);
    assertEquals(1,ret.length);
    assertEquals(FAKE_BLOCK.getLocalBlock(),ret[0].getBlock());
  }
  finally {
    bpos.stop();
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.TestBlockReplacement

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testBlockReplacement() throws Exception {
  final Configuration CONF=new HdfsConfiguration();
  final String[] INITIAL_RACKS={"/RACK0","/RACK1","/RACK2"};
  final String[] NEW_RACKS={"/RACK2"};
  final short REPLICATION_FACTOR=(short)3;
  final int DEFAULT_BLOCK_SIZE=1024;
  final Random r=new Random();
  CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,DEFAULT_BLOCK_SIZE);
  CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY,DEFAULT_BLOCK_SIZE / 2);
  CONF.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,500);
  cluster=new MiniDFSCluster.Builder(CONF).numDataNodes(REPLICATION_FACTOR).racks(INITIAL_RACKS).build();
  try {
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    Path fileName=new Path("/tmp.txt");
    DFSTestUtil.createFile(fs,fileName,DEFAULT_BLOCK_SIZE,REPLICATION_FACTOR,r.nextLong());
    DFSTestUtil.waitReplication(fs,fileName,REPLICATION_FACTOR);
    InetSocketAddress addr=new InetSocketAddress("localhost",cluster.getNameNodePort());
    DFSClient client=new DFSClient(addr,CONF);
    List locatedBlocks=client.getNamenode().getBlockLocations("/tmp.txt",0,DEFAULT_BLOCK_SIZE).getLocatedBlocks();
    assertEquals(1,locatedBlocks.size());
    LocatedBlock block=locatedBlocks.get(0);
    DatanodeInfo[] oldNodes=block.getLocations();
    assertEquals(oldNodes.length,3);
    ExtendedBlock b=block.getBlock();
    cluster.startDataNodes(CONF,1,true,null,NEW_RACKS);
    cluster.waitActive();
    DatanodeInfo[] datanodes=client.datanodeReport(DatanodeReportType.ALL);
    DatanodeInfo newNode=null;
    for (    DatanodeInfo node : datanodes) {
      Boolean isNewNode=true;
      for (      DatanodeInfo oldNode : oldNodes) {
        if (node.equals(oldNode)) {
          isNewNode=false;
          break;
        }
      }
      if (isNewNode) {
        newNode=node;
        break;
      }
    }
    assertTrue(newNode != null);
    DatanodeInfo source=null;
    ArrayList proxies=new ArrayList(2);
    for (    DatanodeInfo node : datanodes) {
      if (node != newNode) {
        if (node.getNetworkLocation().equals(newNode.getNetworkLocation())) {
          source=node;
        }
 else {
          proxies.add(node);
        }
      }
    }
    assertTrue(source != null && proxies.size() == 2);
    LOG.info("Testcase 1: Proxy " + newNode + " does not contain the block "+ b);
    assertFalse(replaceBlock(b,source,newNode,proxies.get(0)));
    LOG.info("Testcase 2: Destination " + proxies.get(1) + " contains the block "+ b);
    assertFalse(replaceBlock(b,source,proxies.get(0),proxies.get(1)));
    LOG.info("Testcase 3: Source=" + source + " Proxy="+ proxies.get(0)+ " Destination="+ newNode);
    assertTrue(replaceBlock(b,source,proxies.get(0),newNode));
    checkBlocks(new DatanodeInfo[]{newNode,proxies.get(0),proxies.get(1)},fileName.toString(),DEFAULT_BLOCK_SIZE,REPLICATION_FACTOR,client);
    LOG.info("Testcase 4: invalid del hint " + proxies.get(0));
    assertTrue(replaceBlock(b,proxies.get(0),proxies.get(1),source));
    checkBlocks(proxies.toArray(new DatanodeInfo[proxies.size()]),fileName.toString(),DEFAULT_BLOCK_SIZE,REPLICATION_FACTOR,client);
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.TestCachingStrategy

APIUtilityVerifier InternalCallVerifier NullVerifier 

@Test(timeout=120000) public void testFadviseAfterWriteThenRead() throws Exception {
  LOG.info("testFadviseAfterWriteThenRead");
  tracker.clear();
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  String TEST_PATH="/test";
  int TEST_PATH_LEN=MAX_TEST_FILE_LEN;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    createHdfsFile(fs,new Path(TEST_PATH),TEST_PATH_LEN,true);
    ExtendedBlock block=cluster.getNameNode().getRpcServer().getBlockLocations(TEST_PATH,0,Long.MAX_VALUE).get(0).getBlock();
    String fadvisedFileName=MiniDFSCluster.getBlockFile(0,block).getName();
    Stats stats=tracker.getStats(fadvisedFileName);
    stats.assertDroppedInRange(0,TEST_PATH_LEN - WRITE_PACKET_SIZE);
    stats.clear();
    readHdfsFile(fs,new Path(TEST_PATH),Long.MAX_VALUE,true);
    Assert.assertNotNull(stats);
    stats.assertDroppedInRange(0,TEST_PATH_LEN - WRITE_PACKET_SIZE);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier NullVerifier 

@Test(timeout=120000) public void testNoFadviseAfterWriteThenRead() throws Exception {
  LOG.info("testNoFadviseAfterWriteThenRead");
  tracker.clear();
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  String TEST_PATH="/test";
  int TEST_PATH_LEN=MAX_TEST_FILE_LEN;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    createHdfsFile(fs,new Path(TEST_PATH),TEST_PATH_LEN,false);
    ExtendedBlock block=cluster.getNameNode().getRpcServer().getBlockLocations(TEST_PATH,0,Long.MAX_VALUE).get(0).getBlock();
    String fadvisedFileName=MiniDFSCluster.getBlockFile(0,block).getName();
    Stats stats=tracker.getStats(fadvisedFileName);
    Assert.assertNull(stats);
    readHdfsFile(fs,new Path(TEST_PATH),Long.MAX_VALUE,false);
    Assert.assertNull(stats);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier NullVerifier 

/** 
 * Test the scenario where the DataNode defaults to not dropping the cache,
 * but our client defaults are set.
 */
@Test(timeout=120000) public void testClientDefaults() throws Exception {
  LOG.info("testClientDefaults");
  tracker.clear();
  Configuration conf=new HdfsConfiguration();
  conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_READS_KEY,false);
  conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DROP_CACHE_BEHIND_WRITES_KEY,false);
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS,true);
  conf.setBoolean(DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES,true);
  MiniDFSCluster cluster=null;
  String TEST_PATH="/test";
  int TEST_PATH_LEN=MAX_TEST_FILE_LEN;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    createHdfsFile(fs,new Path(TEST_PATH),TEST_PATH_LEN,null);
    ExtendedBlock block=cluster.getNameNode().getRpcServer().getBlockLocations(TEST_PATH,0,Long.MAX_VALUE).get(0).getBlock();
    String fadvisedFileName=MiniDFSCluster.getBlockFile(0,block).getName();
    Stats stats=tracker.getStats(fadvisedFileName);
    stats.assertDroppedInRange(0,TEST_PATH_LEN - WRITE_PACKET_SIZE);
    stats.clear();
    readHdfsFile(fs,new Path(TEST_PATH),Long.MAX_VALUE,null);
    Assert.assertNotNull(stats);
    stats.assertDroppedInRange(0,TEST_PATH_LEN - WRITE_PACKET_SIZE);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.TestDataDirs

APIUtilityVerifier UtilityVerifier InternalCallVerifier ConditionMatcher HybridVerifier 

@Test(timeout=30000) public void testDataDirParsing() throws Throwable {
  Configuration conf=new Configuration();
  List locations;
  File dir0=new File("/dir0");
  File dir1=new File("/dir1");
  File dir2=new File("/dir2");
  File dir3=new File("/dir3");
  String locations1="[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3";
  conf.set(DFS_DATANODE_DATA_DIR_KEY,locations1);
  locations=DataNode.getStorageLocations(conf);
  assertThat(locations.size(),is(4));
  assertThat(locations.get(0).getStorageType(),is(StorageType.DISK));
  assertThat(locations.get(0).getUri(),is(dir0.toURI()));
  assertThat(locations.get(1).getStorageType(),is(StorageType.DISK));
  assertThat(locations.get(1).getUri(),is(dir1.toURI()));
  assertThat(locations.get(2).getStorageType(),is(StorageType.SSD));
  assertThat(locations.get(2).getUri(),is(dir2.toURI()));
  assertThat(locations.get(3).getStorageType(),is(StorageType.DISK));
  assertThat(locations.get(3).getUri(),is(dir3.toURI()));
  String locations2="[BadMediaType]/dir0,[ssd]/dir1,[disk]/dir2";
  conf.set(DFS_DATANODE_DATA_DIR_KEY,locations2);
  try {
    locations=DataNode.getStorageLocations(conf);
    fail();
  }
 catch (  IllegalArgumentException iae) {
    DataNode.LOG.info("The exception is expected.",iae);
  }
  String locations3="/dir0,/dir1";
  conf.set(DFS_DATANODE_DATA_DIR_KEY,locations3);
  locations=DataNode.getStorageLocations(conf);
  assertThat(locations.size(),is(2));
  assertThat(locations.get(0).getStorageType(),is(StorageType.DISK));
  assertThat(locations.get(0).getUri(),is(dir0.toURI()));
  assertThat(locations.get(1).getStorageType(),is(StorageType.DISK));
  assertThat(locations.get(1).getUri(),is(dir1.toURI()));
}

APIUtilityVerifier EqualityVerifier ConditionMatcher HybridVerifier 

@Test(timeout=30000) public void testDataDirValidation() throws Throwable {
  DataNodeDiskChecker diskChecker=mock(DataNodeDiskChecker.class);
  doThrow(new IOException()).doThrow(new IOException()).doNothing().when(diskChecker).checkDir(any(LocalFileSystem.class),any(Path.class));
  LocalFileSystem fs=mock(LocalFileSystem.class);
  AbstractList locations=new ArrayList();
  locations.add(StorageLocation.parse("file:/p1/"));
  locations.add(StorageLocation.parse("file:/p2/"));
  locations.add(StorageLocation.parse("file:/p3/"));
  List checkedLocations=DataNode.checkStorageLocations(locations,fs,diskChecker);
  assertEquals("number of valid data dirs",1,checkedLocations.size());
  String validDir=checkedLocations.iterator().next().getFile().getPath();
  assertThat("p3 should be valid",new File("/p3/").getPath(),is(validDir));
}

Class: org.apache.hadoop.hdfs.server.datanode.TestDataNodeMXBean

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testDataNodeMXBean() throws Exception {
  Configuration conf=new Configuration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  try {
    List datanodes=cluster.getDataNodes();
    Assert.assertEquals(datanodes.size(),1);
    DataNode datanode=datanodes.get(0);
    MBeanServer mbs=ManagementFactory.getPlatformMBeanServer();
    ObjectName mxbeanName=new ObjectName("Hadoop:service=DataNode,name=DataNodeInfo");
    String clusterId=(String)mbs.getAttribute(mxbeanName,"ClusterId");
    Assert.assertEquals(datanode.getClusterId(),clusterId);
    String version=(String)mbs.getAttribute(mxbeanName,"Version");
    Assert.assertEquals(datanode.getVersion(),version);
    String rpcPort=(String)mbs.getAttribute(mxbeanName,"RpcPort");
    Assert.assertEquals(datanode.getRpcPort(),rpcPort);
    String httpPort=(String)mbs.getAttribute(mxbeanName,"HttpPort");
    Assert.assertEquals(datanode.getHttpPort(),httpPort);
    String namenodeAddresses=(String)mbs.getAttribute(mxbeanName,"NamenodeAddresses");
    Assert.assertEquals(datanode.getNamenodeAddresses(),namenodeAddresses);
    String volumeInfo=(String)mbs.getAttribute(mxbeanName,"VolumeInfo");
    Assert.assertEquals(replaceDigits(datanode.getVolumeInfo()),replaceDigits(volumeInfo));
    int xceiverCount=(Integer)mbs.getAttribute(mxbeanName,"XceiverCount");
    Assert.assertEquals(datanode.getXceiverCount(),xceiverCount);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.TestDataNodeMultipleRegistrations

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * starts single nn and single dn and verifies registration and handshake
 * @throws IOException
 */
@Test public void testFedSingleNN() throws IOException {
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).nameNodePort(9927).build();
  try {
    NameNode nn1=cluster.getNameNode();
    assertNotNull("cannot create nn1",nn1);
    String bpid1=FSImageTestUtil.getFSImage(nn1).getBlockPoolID();
    String cid1=FSImageTestUtil.getFSImage(nn1).getClusterID();
    int lv1=FSImageTestUtil.getFSImage(nn1).getLayoutVersion();
    LOG.info("nn1: lv=" + lv1 + ";cid="+ cid1+ ";bpid="+ bpid1+ ";uri="+ nn1.getNameNodeAddress());
    DataNode dn=cluster.getDataNodes().get(0);
    final Map volInfos=dn.data.getVolumeInfoMap();
    Assert.assertTrue("No volumes in the fsdataset",volInfos.size() > 0);
    int i=0;
    for (    Map.Entry e : volInfos.entrySet()) {
      LOG.info("vol " + i++ + ") "+ e.getKey()+ ": "+ e.getValue());
    }
    assertEquals("number of volumes is wrong",2,volInfos.size());
    for (    BPOfferService bpos : dn.getAllBpOs()) {
      LOG.info("reg: bpid=" + "; name=" + bpos.bpRegistration + "; sid="+ bpos.bpRegistration.getDatanodeUuid()+ "; nna="+ getNNSocketAddress(bpos));
    }
    BPOfferService bpos1=dn.getAllBpOs()[0];
    bpos1.triggerBlockReportForTests();
    assertEquals("wrong nn address",getNNSocketAddress(bpos1),nn1.getNameNodeAddress());
    assertEquals("wrong bpid",bpos1.getBlockPoolId(),bpid1);
    assertEquals("wrong cid",dn.getClusterId(),cid1);
    cluster.shutdown();
    assertEquals(0,dn.getAllBpOs().length);
    cluster=null;
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier NullVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * start multiple NNs and single DN and verifies per BP registrations and
 * handshakes.
 * @throws IOException
 */
@Test public void test2NNRegistration() throws IOException {
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)).build();
  try {
    cluster.waitActive();
    NameNode nn1=cluster.getNameNode(0);
    NameNode nn2=cluster.getNameNode(1);
    assertNotNull("cannot create nn1",nn1);
    assertNotNull("cannot create nn2",nn2);
    String bpid1=FSImageTestUtil.getFSImage(nn1).getBlockPoolID();
    String bpid2=FSImageTestUtil.getFSImage(nn2).getBlockPoolID();
    String cid1=FSImageTestUtil.getFSImage(nn1).getClusterID();
    String cid2=FSImageTestUtil.getFSImage(nn2).getClusterID();
    int lv1=FSImageTestUtil.getFSImage(nn1).getLayoutVersion();
    int lv2=FSImageTestUtil.getFSImage(nn2).getLayoutVersion();
    int ns1=FSImageTestUtil.getFSImage(nn1).getNamespaceID();
    int ns2=FSImageTestUtil.getFSImage(nn2).getNamespaceID();
    assertNotSame("namespace ids should be different",ns1,ns2);
    LOG.info("nn1: lv=" + lv1 + ";cid="+ cid1+ ";bpid="+ bpid1+ ";uri="+ nn1.getNameNodeAddress());
    LOG.info("nn2: lv=" + lv2 + ";cid="+ cid2+ ";bpid="+ bpid2+ ";uri="+ nn2.getNameNodeAddress());
    DataNode dn=cluster.getDataNodes().get(0);
    final Map volInfos=dn.data.getVolumeInfoMap();
    Assert.assertTrue("No volumes in the fsdataset",volInfos.size() > 0);
    int i=0;
    for (    Map.Entry e : volInfos.entrySet()) {
      LOG.info("vol " + i++ + ") "+ e.getKey()+ ": "+ e.getValue());
    }
    assertEquals("number of volumes is wrong",2,volInfos.size());
    for (    BPOfferService bpos : dn.getAllBpOs()) {
      LOG.info("BP: " + bpos);
    }
    BPOfferService bpos1=dn.getAllBpOs()[0];
    BPOfferService bpos2=dn.getAllBpOs()[1];
    if (getNNSocketAddress(bpos1).equals(nn2.getNameNodeAddress())) {
      BPOfferService tmp=bpos1;
      bpos1=bpos2;
      bpos2=tmp;
    }
    assertEquals("wrong nn address",getNNSocketAddress(bpos1),nn1.getNameNodeAddress());
    assertEquals("wrong nn address",getNNSocketAddress(bpos2),nn2.getNameNodeAddress());
    assertEquals("wrong bpid",bpos1.getBlockPoolId(),bpid1);
    assertEquals("wrong bpid",bpos2.getBlockPoolId(),bpid2);
    assertEquals("wrong cid",dn.getClusterId(),cid1);
    assertEquals("cid should be same",cid2,cid1);
    assertEquals("namespace should be same",bpos1.bpNSInfo.namespaceID,ns1);
    assertEquals("namespace should be same",bpos2.bpNSInfo.namespaceID,ns2);
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.TestDataNodeRollingUpgrade

APIUtilityVerifier InternalCallVerifier ConditionMatcher 

@Test(timeout=600000) public void testDatanodeRollingUpgradeWithRollback() throws Exception {
  try {
    startCluster();
    Path testFile1=new Path("/TestDataNodeRollingUpgrade1.dat");
    DFSTestUtil.createFile(fs,testFile1,FILE_SIZE,REPL_FACTOR,SEED);
    String fileContents1=DFSTestUtil.readFile(fs,testFile1);
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    startRollingUpgrade();
    File blockFile=getBlockForFile(testFile1,true);
    File trashFile=getTrashFileForBlock(blockFile,false);
    deleteAndEnsureInTrash(testFile1,blockFile,trashFile);
    rollbackRollingUpgrade();
    ensureTrashRestored(blockFile,trashFile);
    assert (fs.exists(testFile1));
    String fileContents2=DFSTestUtil.readFile(fs,testFile1);
    assertThat(fileContents1,is(fileContents2));
  }
  finally {
    shutdownCluster();
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.TestDataNodeVolumeFailureToleration

APIUtilityVerifier InternalCallVerifier BooleanVerifier AssumptionSetter HybridVerifier 

/** 
 * Test the DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY configuration
 * option, ie the DN tolerates a failed-to-use scenario during
 * its start-up.
 */
@Test public void testValidVolumesAtStartup() throws Exception {
  assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
  cluster.shutdownDataNodes();
  conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,1);
  File tld=new File(MiniDFSCluster.getBaseDirectory(),"badData");
  File dataDir1=new File(tld,"data1");
  File dataDir1Actual=new File(dataDir1,"1");
  dataDir1Actual.mkdirs();
  File dataDir2=new File(tld,"data2");
  prepareDirToFail(dataDir2);
  File dataDir2Actual=new File(dataDir2,"2");
  conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,dataDir1Actual.getPath() + "," + dataDir2Actual.getPath());
  cluster.startDataNodes(conf,1,false,null,null);
  cluster.waitActive();
  try {
    assertTrue("The DN should have started up fine.",cluster.isDataNodeUp());
    DataNode dn=cluster.getDataNodes().get(0);
    String si=DataNodeTestUtils.getFSDataset(dn).getStorageInfo();
    assertTrue("The DN should have started with this directory",si.contains(dataDir1Actual.getPath()));
    assertFalse("The DN shouldn't have a bad directory.",si.contains(dataDir2Actual.getPath()));
  }
  finally {
    cluster.shutdownDataNodes();
    FileUtil.chmod(dataDir2.toString(),"755");
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.TestDiskError

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Test to check that a DN goes down when all its volumes have failed.
 */
@Test public void testShutdown() throws Exception {
  if (System.getProperty("os.name").startsWith("Windows")) {
    return;
  }
  cluster.startDataNodes(conf,2,true,null,null);
  cluster.waitActive();
  final int dnIndex=0;
  String bpid=cluster.getNamesystem().getBlockPoolId();
  File storageDir=cluster.getInstanceStorageDir(dnIndex,0);
  File dir1=MiniDFSCluster.getRbwDir(storageDir,bpid);
  storageDir=cluster.getInstanceStorageDir(dnIndex,1);
  File dir2=MiniDFSCluster.getRbwDir(storageDir,bpid);
  try {
    assertTrue("Couldn't chmod local vol",dir1.setReadOnly());
    assertTrue("Couldn't chmod local vol",dir2.setReadOnly());
    DataNode dn=cluster.getDataNodes().get(dnIndex);
    for (int i=0; dn.isDatanodeUp(); i++) {
      Path fileName=new Path("/test.txt" + i);
      DFSTestUtil.createFile(fs,fileName,1024,(short)2,1L);
      DFSTestUtil.waitReplication(fs,fileName,(short)2);
      fs.delete(fileName,true);
    }
  }
  finally {
    FileUtil.setWritable(dir1,true);
    FileUtil.setWritable(dir2,true);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Check that the permissions of the local DN directories are as expected.
 */
@Test public void testLocalDirs() throws Exception {
  Configuration conf=new Configuration();
  final String permStr=conf.get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_PERMISSION_KEY);
  FsPermission expected=new FsPermission(permStr);
  FileSystem localFS=FileSystem.getLocal(conf);
  for (  DataNode dn : cluster.getDataNodes()) {
    for (    FsVolumeSpi v : dn.getFSDataset().getVolumes()) {
      String dir=v.getBasePath();
      Path dataDir=new Path(dir);
      FsPermission actual=localFS.getFileStatus(dataDir).getPermission();
      assertEquals("Permission for dir: " + dataDir + ", is "+ actual+ ", while expected is "+ expected,expected,actual);
    }
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.TestIncrementalBrVariations

APIUtilityVerifier InternalCallVerifier NullVerifier 

/** 
 * Verify that the NameNode can learn about new storages from incremental
 * block reports.
 * This tests the fix for the error condition seen in HDFS-6904.
 * @throws IOException
 * @throws InterruptedException
 */
@Test(timeout=60000) public void testNnLearnsNewStorages() throws IOException, InterruptedException {
  final String newStorageUuid=UUID.randomUUID().toString();
  final DatanodeStorage newStorage=new DatanodeStorage(newStorageUuid);
  StorageReceivedDeletedBlocks[] reports=makeReportForReceivedBlock(getDummyBlock(),newStorage);
  cluster.getNameNodeRpc().blockReceivedAndDeleted(dn0Reg,poolId,reports);
  DatanodeStorageInfo storageInfo=cluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getDatanode(dn0.getDatanodeId()).getStorageInfo(newStorageUuid);
  assertNotNull(storageInfo);
}

Class: org.apache.hadoop.hdfs.server.datanode.TestReadOnlySharedStorage

APIUtilityVerifier TestInitializer InternalCallVerifier ConditionMatcher HybridVerifier 

/** 
 * Setup a {@link MiniDFSCluster}.
 * Create a block with both {@link State#NORMAL} and {@link State#READ_ONLY_SHARED} replicas.
 */
@Before public void setup() throws IOException, InterruptedException {
  conf=new HdfsConfiguration();
  SimulatedFSDataset.setFactory(conf);
  Configuration[] overlays=new Configuration[NUM_DATANODES];
  for (int i=0; i < overlays.length; i++) {
    overlays[i]=new Configuration();
    if (i == RO_NODE_INDEX) {
      overlays[i].setEnum(SimulatedFSDataset.CONFIG_PROPERTY_STATE,i == RO_NODE_INDEX ? READ_ONLY_SHARED : NORMAL);
    }
  }
  cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).dataNodeConfOverlays(overlays).build();
  fs=cluster.getFileSystem();
  blockManager=cluster.getNameNode().getNamesystem().getBlockManager();
  datanodeManager=blockManager.getDatanodeManager();
  client=new DFSClient(new InetSocketAddress("localhost",cluster.getNameNodePort()),cluster.getConfiguration(0));
  for (int i=0; i < NUM_DATANODES; i++) {
    DataNode dataNode=cluster.getDataNodes().get(i);
    validateStorageState(BlockManagerTestUtil.getStorageReportsForDatanode(datanodeManager.getDatanode(dataNode.getDatanodeId())),i == RO_NODE_INDEX ? READ_ONLY_SHARED : NORMAL);
  }
  DFSTestUtil.createFile(fs,PATH,BLOCK_SIZE,BLOCK_SIZE,BLOCK_SIZE,(short)1,seed);
  LocatedBlock locatedBlock=getLocatedBlock();
  extendedBlock=locatedBlock.getBlock();
  block=extendedBlock.getLocalBlock();
  assertThat(locatedBlock.getLocations().length,is(1));
  normalDataNode=locatedBlock.getLocations()[0];
  readOnlyDataNode=datanodeManager.getDatanode(cluster.getDataNodes().get(RO_NODE_INDEX).getDatanodeId());
  assertThat(normalDataNode,is(not(readOnlyDataNode)));
  validateNumberReplicas(1);
  cluster.injectBlocks(0,RO_NODE_INDEX,Collections.singleton(block));
  waitForLocations(2);
}

Class: org.apache.hadoop.hdfs.server.datanode.TestSimulatedFSDataset

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testInvalidate() throws IOException {
  final SimulatedFSDataset fsdataset=getSimulatedFSDataset();
  int bytesAdded=addSomeBlocks(fsdataset);
  Block[] deleteBlocks=new Block[2];
  deleteBlocks[0]=new Block(1,0,0);
  deleteBlocks[1]=new Block(2,0,0);
  fsdataset.invalidate(bpid,deleteBlocks);
  checkInvalidBlock(new ExtendedBlock(bpid,deleteBlocks[0]));
  checkInvalidBlock(new ExtendedBlock(bpid,deleteBlocks[1]));
  long sizeDeleted=blockIdToLen(1) + blockIdToLen(2);
  assertEquals(bytesAdded - sizeDeleted,fsdataset.getDfsUsed());
  assertEquals(fsdataset.getCapacity() - bytesAdded + sizeDeleted,fsdataset.getRemaining());
  for (int i=3; i <= NUMBLOCKS; ++i) {
    Block b=new Block(i,0,0);
    assertTrue(fsdataset.isValidBlock(new ExtendedBlock(bpid,b)));
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testGetMetaData() throws IOException {
  final SimulatedFSDataset fsdataset=getSimulatedFSDataset();
  ExtendedBlock b=new ExtendedBlock(bpid,1,5,0);
  try {
    assertTrue(fsdataset.getMetaDataInputStream(b) == null);
    assertTrue("Expected an IO exception",false);
  }
 catch (  IOException e) {
  }
  addSomeBlocks(fsdataset);
  b=new ExtendedBlock(bpid,1,0,0);
  InputStream metaInput=fsdataset.getMetaDataInputStream(b);
  DataInputStream metaDataInput=new DataInputStream(metaInput);
  short version=metaDataInput.readShort();
  assertEquals(BlockMetadataHeader.VERSION,version);
  DataChecksum checksum=DataChecksum.newDataChecksum(metaDataInput);
  assertEquals(DataChecksum.Type.NULL,checksum.getChecksumType());
  assertEquals(0,checksum.getChecksumSize());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testInjectionNonEmpty() throws IOException {
  SimulatedFSDataset fsdataset=getSimulatedFSDataset();
  BlockListAsLongs blockReport=fsdataset.getBlockReport(bpid);
  assertEquals(0,blockReport.getNumberOfBlocks());
  int bytesAdded=addSomeBlocks(fsdataset);
  blockReport=fsdataset.getBlockReport(bpid);
  assertEquals(NUMBLOCKS,blockReport.getNumberOfBlocks());
  for (  Block b : blockReport) {
    assertNotNull(b);
    assertEquals(blockIdToLen(b.getBlockId()),b.getNumBytes());
  }
  fsdataset=null;
  SimulatedFSDataset sfsdataset=getSimulatedFSDataset();
  bytesAdded+=addSomeBlocks(sfsdataset,NUMBLOCKS + 1);
  sfsdataset.getBlockReport(bpid);
  assertEquals(NUMBLOCKS,blockReport.getNumberOfBlocks());
  sfsdataset.getBlockReport(bpid);
  assertEquals(NUMBLOCKS,blockReport.getNumberOfBlocks());
  sfsdataset.injectBlocks(bpid,blockReport);
  blockReport=sfsdataset.getBlockReport(bpid);
  assertEquals(NUMBLOCKS * 2,blockReport.getNumberOfBlocks());
  for (  Block b : blockReport) {
    assertNotNull(b);
    assertEquals(blockIdToLen(b.getBlockId()),b.getNumBytes());
    assertEquals(blockIdToLen(b.getBlockId()),sfsdataset.getLength(new ExtendedBlock(bpid,b)));
  }
  assertEquals(bytesAdded,sfsdataset.getDfsUsed());
  assertEquals(sfsdataset.getCapacity() - bytesAdded,sfsdataset.getRemaining());
  conf.setLong(SimulatedFSDataset.CONFIG_PROPERTY_CAPACITY,10);
  try {
    sfsdataset=getSimulatedFSDataset();
    sfsdataset.addBlockPool(bpid,conf);
    sfsdataset.injectBlocks(bpid,blockReport);
    assertTrue("Expected an IO exception",false);
  }
 catch (  IOException e) {
  }
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testInjectionEmpty() throws IOException {
  SimulatedFSDataset fsdataset=getSimulatedFSDataset();
  BlockListAsLongs blockReport=fsdataset.getBlockReport(bpid);
  assertEquals(0,blockReport.getNumberOfBlocks());
  int bytesAdded=addSomeBlocks(fsdataset);
  blockReport=fsdataset.getBlockReport(bpid);
  assertEquals(NUMBLOCKS,blockReport.getNumberOfBlocks());
  for (  Block b : blockReport) {
    assertNotNull(b);
    assertEquals(blockIdToLen(b.getBlockId()),b.getNumBytes());
  }
  SimulatedFSDataset sfsdataset=getSimulatedFSDataset();
  sfsdataset.injectBlocks(bpid,blockReport);
  blockReport=sfsdataset.getBlockReport(bpid);
  assertEquals(NUMBLOCKS,blockReport.getNumberOfBlocks());
  for (  Block b : blockReport) {
    assertNotNull(b);
    assertEquals(blockIdToLen(b.getBlockId()),b.getNumBytes());
    assertEquals(blockIdToLen(b.getBlockId()),sfsdataset.getLength(new ExtendedBlock(bpid,b)));
  }
  assertEquals(bytesAdded,sfsdataset.getDfsUsed());
  assertEquals(sfsdataset.getCapacity() - bytesAdded,sfsdataset.getRemaining());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testStorageUsage() throws IOException {
  final SimulatedFSDataset fsdataset=getSimulatedFSDataset();
  assertEquals(fsdataset.getDfsUsed(),0);
  assertEquals(fsdataset.getRemaining(),fsdataset.getCapacity());
  int bytesAdded=addSomeBlocks(fsdataset);
  assertEquals(bytesAdded,fsdataset.getDfsUsed());
  assertEquals(fsdataset.getCapacity() - bytesAdded,fsdataset.getRemaining());
}

Class: org.apache.hadoop.hdfs.server.datanode.TestTransferRbw

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testTransferRbw() throws Exception {
  final HdfsConfiguration conf=new HdfsConfiguration();
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION).build();
  try {
    cluster.waitActive();
    final DistributedFileSystem fs=cluster.getFileSystem();
    final Path p=new Path("/foo");
    final int size=(1 << 16) + RAN.nextInt(1 << 16);
    LOG.info("size = " + size);
    final FSDataOutputStream out=fs.create(p,REPLICATION);
    final byte[] bytes=new byte[1024];
    for (int remaining=size; remaining > 0; ) {
      RAN.nextBytes(bytes);
      final int len=bytes.length < remaining ? bytes.length : remaining;
      out.write(bytes,0,len);
      out.hflush();
      remaining-=len;
    }
    final ReplicaBeingWritten oldrbw;
    final DataNode newnode;
    final DatanodeInfo newnodeinfo;
    final String bpid=cluster.getNamesystem().getBlockPoolId();
{
      final DataNode oldnode=cluster.getDataNodes().get(0);
      oldrbw=getRbw(oldnode,bpid);
      LOG.info("oldrbw = " + oldrbw);
      cluster.startDataNodes(conf,1,true,null,null);
      newnode=cluster.getDataNodes().get(REPLICATION);
      final DatanodeInfo oldnodeinfo;
{
        final DatanodeInfo[] datatnodeinfos=cluster.getNameNodeRpc().getDatanodeReport(DatanodeReportType.LIVE);
        Assert.assertEquals(2,datatnodeinfos.length);
        int i=0;
        for (DatanodeRegistration dnReg=newnode.getDNRegistrationForBP(bpid); i < datatnodeinfos.length && !datatnodeinfos[i].equals(dnReg); i++)         ;
        Assert.assertTrue(i < datatnodeinfos.length);
        newnodeinfo=datatnodeinfos[i];
        oldnodeinfo=datatnodeinfos[1 - i];
      }
      final ExtendedBlock b=new ExtendedBlock(bpid,oldrbw.getBlockId(),oldrbw.getBytesAcked(),oldrbw.getGenerationStamp());
      final BlockOpResponseProto s=DFSTestUtil.transferRbw(b,DFSClientAdapter.getDFSClient(fs),oldnodeinfo,newnodeinfo);
      Assert.assertEquals(Status.SUCCESS,s.getStatus());
    }
    final ReplicaBeingWritten newrbw=getRbw(newnode,bpid);
    LOG.info("newrbw = " + newrbw);
    Assert.assertEquals(oldrbw.getBlockId(),newrbw.getBlockId());
    Assert.assertEquals(oldrbw.getGenerationStamp(),newrbw.getGenerationStamp());
    Assert.assertEquals(oldrbw.getVisibleLength(),newrbw.getVisibleLength());
    LOG.info("DONE");
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.TestDatanodeRestart

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier 

@Test public void testRecoverReplicas() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,1024L);
  conf.setInt(DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY,512);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  cluster.waitActive();
  try {
    FileSystem fs=cluster.getFileSystem();
    for (int i=0; i < 4; i++) {
      Path fileName=new Path("/test" + i);
      DFSTestUtil.createFile(fs,fileName,1,(short)1,0L);
      DFSTestUtil.waitReplication(fs,fileName,(short)1);
    }
    String bpid=cluster.getNamesystem().getBlockPoolId();
    DataNode dn=cluster.getDataNodes().get(0);
    Iterator replicasItor=dataset(dn).volumeMap.replicas(bpid).iterator();
    ReplicaInfo replica=replicasItor.next();
    createUnlinkTmpFile(replica,true,true);
    createUnlinkTmpFile(replica,false,true);
    replica=replicasItor.next();
    createUnlinkTmpFile(replica,true,false);
    createUnlinkTmpFile(replica,false,false);
    replica=replicasItor.next();
    createUnlinkTmpFile(replica,true,true);
    createUnlinkTmpFile(replica,false,false);
    cluster.restartDataNodes();
    cluster.waitActive();
    dn=cluster.getDataNodes().get(0);
    Collection replicas=dataset(dn).volumeMap.replicas(bpid);
    Assert.assertEquals(4,replicas.size());
    replicasItor=replicas.iterator();
    while (replicasItor.hasNext()) {
      Assert.assertEquals(ReplicaState.FINALIZED,replicasItor.next().getState());
    }
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.TestInterDatanodeProtocol

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test  for{@link FsDatasetImpl#updateReplicaUnderRecovery(ExtendedBlock,long,long)} 
 */
@Test public void testUpdateReplicaUnderRecovery() throws IOException {
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    cluster.waitActive();
    String bpid=cluster.getNamesystem().getBlockPoolId();
    DistributedFileSystem dfs=cluster.getFileSystem();
    String filestr="/foo";
    Path filepath=new Path(filestr);
    DFSTestUtil.createFile(dfs,filepath,1024L,(short)3,0L);
    final LocatedBlock locatedblock=getLastLocatedBlock(DFSClientAdapter.getDFSClient(dfs).getNamenode(),filestr);
    final DatanodeInfo[] datanodeinfo=locatedblock.getLocations();
    Assert.assertTrue(datanodeinfo.length > 0);
    final DataNode datanode=cluster.getDataNode(datanodeinfo[0].getIpcPort());
    Assert.assertTrue(datanode != null);
    final ExtendedBlock b=locatedblock.getBlock();
    final long recoveryid=b.getGenerationStamp() + 1;
    final long newlength=b.getNumBytes() - 1;
    final FsDatasetSpi fsdataset=DataNodeTestUtils.getFSDataset(datanode);
    final ReplicaRecoveryInfo rri=fsdataset.initReplicaRecovery(new RecoveringBlock(b,null,recoveryid));
    final ReplicaInfo replica=FsDatasetTestUtil.fetchReplicaInfo(fsdataset,bpid,b.getBlockId());
    Assert.assertEquals(ReplicaState.RUR,replica.getState());
    FsDatasetImpl.checkReplicaFiles(replica);
{
      final ExtendedBlock tmp=new ExtendedBlock(b.getBlockPoolId(),rri.getBlockId(),rri.getNumBytes() - 1,rri.getGenerationStamp());
      try {
        fsdataset.updateReplicaUnderRecovery(tmp,recoveryid,newlength);
        Assert.fail();
      }
 catch (      IOException ioe) {
        System.out.println("GOOD: getting " + ioe);
      }
    }
    final String storageID=fsdataset.updateReplicaUnderRecovery(new ExtendedBlock(b.getBlockPoolId(),rri),recoveryid,newlength);
    assertTrue(storageID != null);
  }
  finally {
    if (cluster != null)     cluster.shutdown();
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test {@link FsDatasetImpl#initReplicaRecovery(String,ReplicaMap,Block,long,long)}
 */
@Test public void testInitReplicaRecovery() throws IOException {
  final long firstblockid=10000L;
  final long gs=7777L;
  final long length=22L;
  final ReplicaMap map=new ReplicaMap(this);
  String bpid="BP-TEST";
  final Block[] blocks=new Block[5];
  for (int i=0; i < blocks.length; i++) {
    blocks[i]=new Block(firstblockid + i,length,gs);
    map.add(bpid,createReplicaInfo(blocks[i]));
  }
{
    final Block b=blocks[0];
    final ReplicaInfo originalInfo=map.get(bpid,b);
    final long recoveryid=gs + 1;
    final ReplicaRecoveryInfo recoveryInfo=FsDatasetImpl.initReplicaRecovery(bpid,map,blocks[0],recoveryid,DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
    assertEquals(originalInfo,recoveryInfo);
    final ReplicaUnderRecovery updatedInfo=(ReplicaUnderRecovery)map.get(bpid,b);
    Assert.assertEquals(originalInfo.getBlockId(),updatedInfo.getBlockId());
    Assert.assertEquals(recoveryid,updatedInfo.getRecoveryID());
    final long recoveryid2=gs + 2;
    final ReplicaRecoveryInfo recoveryInfo2=FsDatasetImpl.initReplicaRecovery(bpid,map,blocks[0],recoveryid2,DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
    assertEquals(originalInfo,recoveryInfo2);
    final ReplicaUnderRecovery updatedInfo2=(ReplicaUnderRecovery)map.get(bpid,b);
    Assert.assertEquals(originalInfo.getBlockId(),updatedInfo2.getBlockId());
    Assert.assertEquals(recoveryid2,updatedInfo2.getRecoveryID());
    try {
      FsDatasetImpl.initReplicaRecovery(bpid,map,b,recoveryid,DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
      Assert.fail();
    }
 catch (    RecoveryInProgressException ripe) {
      System.out.println("GOOD: getting " + ripe);
    }
  }
{
    final long recoveryid=gs + 1;
    final Block b=new Block(firstblockid - 1,length,gs);
    ReplicaRecoveryInfo r=FsDatasetImpl.initReplicaRecovery(bpid,map,b,recoveryid,DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
    Assert.assertNull("Data-node should not have this replica.",r);
  }
{
    final long recoveryid=gs - 1;
    final Block b=new Block(firstblockid + 1,length,gs);
    try {
      FsDatasetImpl.initReplicaRecovery(bpid,map,b,recoveryid,DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
      Assert.fail();
    }
 catch (    IOException ioe) {
      System.out.println("GOOD: getting " + ioe);
    }
  }
{
    final long recoveryid=gs + 1;
    final Block b=new Block(firstblockid,length,gs + 1);
    try {
      FsDatasetImpl.initReplicaRecovery(bpid,map,b,recoveryid,DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_DEFAULT);
      fail("InitReplicaRecovery should fail because replica's " + "gs is less than the block's gs");
    }
 catch (    IOException e) {
      e.getMessage().startsWith("replica.getGenerationStamp() < block.getGenerationStamp(), block=");
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.FSXAttrBaseTest

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

@Test(timeout=120000) public void testXAttrAcl() throws Exception {
  FileSystem.mkdirs(fs,path,FsPermission.createImmutable((short)0750));
  fs.setOwner(path,BRUCE.getUserName(),null);
  FileSystem fsAsBruce=createFileSystem(BRUCE);
  FileSystem fsAsDiana=createFileSystem(DIANA);
  fsAsBruce.setXAttr(path,name1,value1);
  Map xattrs;
  try {
    xattrs=fsAsDiana.getXAttrs(path);
    Assert.fail("Diana should not have read access to get xattrs");
  }
 catch (  AccessControlException e) {
  }
  fsAsBruce.modifyAclEntries(path,Lists.newArrayList(aclEntry(ACCESS,USER,DIANA.getUserName(),READ)));
  xattrs=fsAsDiana.getXAttrs(path);
  Assert.assertArrayEquals(value1,xattrs.get(name1));
  try {
    fsAsDiana.removeXAttr(path,name1);
    Assert.fail("Diana should not have write access to remove xattrs");
  }
 catch (  AccessControlException e) {
  }
  try {
    fsAsDiana.setXAttr(path,name2,value2);
    Assert.fail("Diana should not have write access to set xattrs");
  }
 catch (  AccessControlException e) {
  }
  fsAsBruce.modifyAclEntries(path,Lists.newArrayList(aclEntry(ACCESS,USER,DIANA.getUserName(),ALL)));
  fsAsDiana.setXAttr(path,name2,value2);
  Assert.assertArrayEquals(value2,fsAsDiana.getXAttrs(path).get(name2));
  fsAsDiana.removeXAttr(path,name1);
  fsAsDiana.removeXAttr(path,name2);
}

Class: org.apache.hadoop.hdfs.server.namenode.TestAuditLogger

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testWebHdfsAuditLogger() throws IOException, URISyntaxException {
  Configuration conf=new HdfsConfiguration();
  conf.set(DFS_NAMENODE_AUDIT_LOGGERS_KEY,DummyAuditLogger.class.getName());
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).build();
  GetOpParam.Op op=GetOpParam.Op.GETFILESTATUS;
  try {
    cluster.waitClusterUp();
    assertTrue(DummyAuditLogger.initialized);
    URI uri=new URI("http",NetUtils.getHostPortString(cluster.getNameNode().getHttpAddress()),"/webhdfs/v1/",op.toQueryString(),null);
    HttpURLConnection conn=(HttpURLConnection)uri.toURL().openConnection();
    conn.setRequestMethod(op.getType().toString());
    conn.connect();
    assertEquals(200,conn.getResponseCode());
    conn.disconnect();
    assertEquals(1,DummyAuditLogger.logCount);
    assertEquals("127.0.0.1",DummyAuditLogger.remoteAddr);
    conn=(HttpURLConnection)uri.toURL().openConnection();
    conn.setRequestMethod(op.getType().toString());
    conn.setRequestProperty("X-Forwarded-For","1.1.1.1");
    conn.connect();
    assertEquals(200,conn.getResponseCode());
    conn.disconnect();
    assertEquals(2,DummyAuditLogger.logCount);
    assertEquals("127.0.0.1",DummyAuditLogger.remoteAddr);
    conf.set(ProxyServers.CONF_HADOOP_PROXYSERVERS,"127.0.0.1");
    ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
    conn=(HttpURLConnection)uri.toURL().openConnection();
    conn.setRequestMethod(op.getType().toString());
    conn.setRequestProperty("X-Forwarded-For","1.1.1.1");
    conn.connect();
    assertEquals(200,conn.getResponseCode());
    conn.disconnect();
    assertEquals(3,DummyAuditLogger.logCount);
    assertEquals("1.1.1.1",DummyAuditLogger.remoteAddr);
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestAuditLogs

APIUtilityVerifier UtilityVerifier InternalCallVerifier 

/** 
 * test that denied access via webhdfs puts proper entry in audit log 
 */
@Test public void testAuditWebHdfsDenied() throws Exception {
  final Path file=new Path(fnames[0]);
  fs.setPermission(file,new FsPermission((short)0600));
  fs.setOwner(file,"root",null);
  setupAuditLogs();
  try {
    WebHdfsFileSystem webfs=WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo,conf,WebHdfsFileSystem.SCHEME);
    InputStream istream=webfs.open(file);
    int val=istream.read();
    fail("open+read must not succeed, got " + val);
  }
 catch (  AccessControlException E) {
    System.out.println("got access denied, as expected.");
  }
  verifyAuditLogsRepeat(false,2);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * test that allowed stat puts proper entry in audit log 
 */
@Test public void testAuditAllowedStat() throws Exception {
  final Path file=new Path(fnames[0]);
  FileSystem userfs=DFSTestUtil.getFileSystemAs(userGroupInfo,conf);
  setupAuditLogs();
  FileStatus st=userfs.getFileStatus(file);
  verifyAuditLogs(true);
  assertTrue("failed to stat file",st != null && st.isFile());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * test that stat via webhdfs puts proper entry in audit log 
 */
@Test public void testAuditWebHdfsStat() throws Exception {
  final Path file=new Path(fnames[0]);
  fs.setPermission(file,new FsPermission((short)0644));
  fs.setOwner(file,"root",null);
  setupAuditLogs();
  WebHdfsFileSystem webfs=WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo,conf,WebHdfsFileSystem.SCHEME);
  FileStatus st=webfs.getFileStatus(file);
  verifyAuditLogs(true);
  assertTrue("failed to stat file",st != null && st.isFile());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * test that access via webhdfs puts proper entry in audit log 
 */
@Test public void testAuditWebHdfs() throws Exception {
  final Path file=new Path(fnames[0]);
  fs.setPermission(file,new FsPermission((short)0644));
  fs.setOwner(file,"root",null);
  setupAuditLogs();
  WebHdfsFileSystem webfs=WebHdfsTestUtil.getWebHdfsFileSystemAs(userGroupInfo,conf,WebHdfsFileSystem.SCHEME);
  InputStream istream=webfs.open(file);
  int val=istream.read();
  istream.close();
  verifyAuditLogsRepeat(true,3);
  assertTrue("failed to read from file",val >= 0);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * test that allowed operation puts proper entry in audit log 
 */
@Test public void testAuditAllowed() throws Exception {
  final Path file=new Path(fnames[0]);
  FileSystem userfs=DFSTestUtil.getFileSystemAs(userGroupInfo,conf);
  setupAuditLogs();
  InputStream istream=userfs.open(file);
  int val=istream.read();
  istream.close();
  verifyAuditLogs(true);
  assertTrue("failed to read from file",val >= 0);
}

Class: org.apache.hadoop.hdfs.server.namenode.TestBackupNode

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Ensure that the backupnode will tail edits from the NN
 * and keep in sync, even while the NN rolls, checkpoints
 * occur, etc.
 */
@Test public void testBackupNodeTailsEdits() throws Exception {
  Configuration conf=new HdfsConfiguration();
  HAUtil.setAllowStandbyReads(conf,true);
  MiniDFSCluster cluster=null;
  FileSystem fileSys=null;
  BackupNode backup=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    fileSys=cluster.getFileSystem();
    backup=startBackupNode(conf,StartupOption.BACKUP,1);
    BackupImage bnImage=(BackupImage)backup.getFSImage();
    testBNInSync(cluster,backup,1);
    NameNode nn=cluster.getNameNode();
    NamenodeProtocols nnRpc=nn.getRpcServer();
    nnRpc.rollEditLog();
    assertEquals(bnImage.getEditLog().getCurSegmentTxId(),nn.getFSImage().getEditLog().getCurSegmentTxId());
    testBNInSync(cluster,backup,2);
    long nnImageBefore=nn.getFSImage().getStorage().getMostRecentCheckpointTxId();
    backup.doCheckpoint();
    long nnImageAfter=nn.getFSImage().getStorage().getMostRecentCheckpointTxId();
    assertTrue("nn should have received new checkpoint. before: " + nnImageBefore + " after: "+ nnImageAfter,nnImageAfter > nnImageBefore);
    testBNInSync(cluster,backup,3);
    StorageDirectory sd=bnImage.getStorage().getStorageDir(0);
    backup.stop();
    backup=null;
    EditLogFile editsLog=FSImageTestUtil.findLatestEditsLog(sd);
    assertEquals(editsLog.getFirstTxId(),nn.getFSImage().getEditLog().getCurSegmentTxId());
    assertTrue("Should not have finalized " + editsLog,editsLog.isInProgress());
    assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down")));
    backup=startBackupNode(conf,StartupOption.BACKUP,1);
    testBNInSync(cluster,backup,4);
    assertNotNull(backup.getNamesystem().getFileInfo("/edit-while-bn-down",false));
    backup.stop(false);
    assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down-2")));
  }
  finally {
    LOG.info("Shutting down...");
    if (backup != null)     backup.stop();
    if (fileSys != null)     fileSys.close();
    if (cluster != null)     cluster.shutdown();
  }
  assertStorageDirsMatch(cluster.getNameNode(),backup);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Verify that a file can be read both from NameNode and BackupNode.
 */
@Test public void testCanReadData() throws IOException {
  Path file1=new Path("/fileToRead.dat");
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  FileSystem fileSys=null;
  BackupNode backup=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).build();
    fileSys=cluster.getFileSystem();
    long txid=cluster.getNameNodeRpc().getTransactionID();
    backup=startBackupNode(conf,StartupOption.BACKUP,1);
    waitCheckpointDone(cluster,txid);
    String rpcAddrKeyPreffix=DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY + ".bnCluster";
    String nnAddr=cluster.getNameNode().getNameNodeAddressHostPortString();
    conf.get(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
    String bnAddr=backup.getNameNodeAddressHostPortString();
    conf.set(DFSConfigKeys.DFS_NAMESERVICES,"bnCluster");
    conf.set(DFSConfigKeys.DFS_NAMESERVICE_ID,"bnCluster");
    conf.set(DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + ".bnCluster","nnActive, nnBackup");
    conf.set(rpcAddrKeyPreffix + ".nnActive",nnAddr);
    conf.set(rpcAddrKeyPreffix + ".nnBackup",bnAddr);
    cluster.startDataNodes(conf,3,true,StartupOption.REGULAR,null);
    DFSTestUtil.createFile(fileSys,file1,fileSize,fileSize,blockSize,(short)3,seed);
    FileSystem bnFS=FileSystem.get(new Path("hdfs://" + bnAddr).toUri(),conf);
    String nnData=DFSTestUtil.readFile(fileSys,file1);
    String bnData=DFSTestUtil.readFile(bnFS,file1);
    assertEquals("Data read from BackupNode and NameNode is not the same.",nnData,bnData);
  }
 catch (  IOException e) {
    LOG.error("Error in TestBackupNode: ",e);
    assertTrue(e.getLocalizedMessage(),false);
  }
 finally {
    if (fileSys != null)     fileSys.close();
    if (backup != null)     backup.stop();
    if (cluster != null)     cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestBlockUnderConstruction

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test NameNode.getBlockLocations(..) on reading un-closed files.
 */
@Test public void testGetBlockLocations() throws IOException {
  final NamenodeProtocols namenode=cluster.getNameNodeRpc();
  final Path p=new Path(BASE_DIR,"file2.dat");
  final String src=p.toString();
  final FSDataOutputStream out=TestFileCreation.createFile(hdfs,p,3);
  int len=BLOCK_SIZE >>> 1;
  writeFile(p,out,len);
  for (int i=1; i < NUM_BLOCKS; ) {
    final LocatedBlocks lb=namenode.getBlockLocations(src,0,len);
    final List blocks=lb.getLocatedBlocks();
    assertEquals(i,blocks.size());
    final Block b=blocks.get(blocks.size() - 1).getBlock().getLocalBlock();
    assertTrue(b instanceof BlockInfoUnderConstruction);
    if (++i < NUM_BLOCKS) {
      writeFile(p,out,BLOCK_SIZE);
      len+=BLOCK_SIZE;
    }
  }
  out.close();
}

Class: org.apache.hadoop.hdfs.server.namenode.TestCacheDirectives

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=60000) public void testCacheManagerRestart() throws Exception {
  SecondaryNameNode secondary=null;
  try {
    conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,"0.0.0.0:0");
    secondary=new SecondaryNameNode(conf);
    final String pool="poolparty";
    String groupName="partygroup";
    FsPermission mode=new FsPermission((short)0777);
    long limit=747;
    dfs.addCachePool(new CachePoolInfo(pool).setGroupName(groupName).setMode(mode).setLimit(limit));
    RemoteIterator pit=dfs.listCachePools();
    assertTrue("No cache pools found",pit.hasNext());
    CachePoolInfo info=pit.next().getInfo();
    assertEquals(pool,info.getPoolName());
    assertEquals(groupName,info.getGroupName());
    assertEquals(mode,info.getMode());
    assertEquals(limit,(long)info.getLimit());
    assertFalse("Unexpected # of cache pools found",pit.hasNext());
    int numEntries=10;
    String entryPrefix="/party-";
    long prevId=-1;
    final Date expiry=new Date();
    for (int i=0; i < numEntries; i++) {
      prevId=dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPath(new Path(entryPrefix + i)).setPool(pool).setExpiration(CacheDirectiveInfo.Expiration.newAbsolute(expiry.getTime())).build());
    }
    RemoteIterator dit=dfs.listCacheDirectives(null);
    for (int i=0; i < numEntries; i++) {
      assertTrue("Unexpected # of cache entries: " + i,dit.hasNext());
      CacheDirectiveInfo cd=dit.next().getInfo();
      assertEquals(i + 1,cd.getId().longValue());
      assertEquals(entryPrefix + i,cd.getPath().toUri().getPath());
      assertEquals(pool,cd.getPool());
    }
    assertFalse("Unexpected # of cache directives found",dit.hasNext());
    secondary.doCheckpoint();
    final String imagePool="imagePool";
    dfs.addCachePool(new CachePoolInfo(imagePool));
    prevId=dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPath(new Path("/image")).setPool(imagePool).build());
    dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    dfs.saveNamespace();
    dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    boolean fetchImage=secondary.doCheckpoint();
    assertTrue("Secondary should have fetched a new fsimage from NameNode",fetchImage);
    dfs.removeCachePool(imagePool);
    cluster.restartNameNode();
    pit=dfs.listCachePools();
    assertTrue("No cache pools found",pit.hasNext());
    info=pit.next().getInfo();
    assertEquals(pool,info.getPoolName());
    assertEquals(pool,info.getPoolName());
    assertEquals(groupName,info.getGroupName());
    assertEquals(mode,info.getMode());
    assertEquals(limit,(long)info.getLimit());
    assertFalse("Unexpected # of cache pools found",pit.hasNext());
    dit=dfs.listCacheDirectives(null);
    for (int i=0; i < numEntries; i++) {
      assertTrue("Unexpected # of cache entries: " + i,dit.hasNext());
      CacheDirectiveInfo cd=dit.next().getInfo();
      assertEquals(i + 1,cd.getId().longValue());
      assertEquals(entryPrefix + i,cd.getPath().toUri().getPath());
      assertEquals(pool,cd.getPool());
      assertEquals(expiry.getTime(),cd.getExpiration().getMillis());
    }
    assertFalse("Unexpected # of cache directives found",dit.hasNext());
    long nextId=dfs.addCacheDirective(new CacheDirectiveInfo.Builder().setPath(new Path("/foobar")).setPool(pool).build());
    assertEquals(prevId + 1,nextId);
  }
  finally {
    if (secondary != null) {
      secondary.shutdown();
    }
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=60000) public void testAddRemoveDirectives() throws Exception {
  proto.addCachePool(new CachePoolInfo("pool1").setMode(new FsPermission((short)0777)));
  proto.addCachePool(new CachePoolInfo("pool2").setMode(new FsPermission((short)0777)));
  proto.addCachePool(new CachePoolInfo("pool3").setMode(new FsPermission((short)0777)));
  proto.addCachePool(new CachePoolInfo("pool4").setMode(new FsPermission((short)0)));
  CacheDirectiveInfo alpha=new CacheDirectiveInfo.Builder().setPath(new Path("/alpha")).setPool("pool1").build();
  CacheDirectiveInfo beta=new CacheDirectiveInfo.Builder().setPath(new Path("/beta")).setPool("pool2").build();
  CacheDirectiveInfo delta=new CacheDirectiveInfo.Builder().setPath(new Path("/delta")).setPool("pool1").build();
  long alphaId=addAsUnprivileged(alpha);
  long alphaId2=addAsUnprivileged(alpha);
  assertFalse("Expected to get unique directives when re-adding an " + "existing CacheDirectiveInfo",alphaId == alphaId2);
  long betaId=addAsUnprivileged(beta);
  try {
    addAsUnprivileged(new CacheDirectiveInfo.Builder().setPath(new Path("/unicorn")).setPool("no_such_pool").build());
    fail("expected an error when adding to a non-existent pool.");
  }
 catch (  InvalidRequestException ioe) {
    GenericTestUtils.assertExceptionContains("Unknown pool",ioe);
  }
  try {
    addAsUnprivileged(new CacheDirectiveInfo.Builder().setPath(new Path("/blackhole")).setPool("pool4").build());
    fail("expected an error when adding to a pool with " + "mode 0 (no permissions for anyone).");
  }
 catch (  AccessControlException e) {
    GenericTestUtils.assertExceptionContains("Permission denied while accessing pool",e);
  }
  try {
    addAsUnprivileged(new CacheDirectiveInfo.Builder().setPath(new Path("/illegal:path/")).setPool("pool1").build());
    fail("expected an error when adding a malformed path " + "to the cache directives.");
  }
 catch (  IllegalArgumentException e) {
    GenericTestUtils.assertExceptionContains("is not a valid DFS filename",e);
  }
  try {
    addAsUnprivileged(new CacheDirectiveInfo.Builder().setPath(new Path("/emptypoolname")).setReplication((short)1).setPool("").build());
    fail("expected an error when adding a cache " + "directive with an empty pool name.");
  }
 catch (  InvalidRequestException e) {
    GenericTestUtils.assertExceptionContains("Invalid empty pool name",e);
  }
  long deltaId=addAsUnprivileged(delta);
  long relativeId=addAsUnprivileged(new CacheDirectiveInfo.Builder().setPath(new Path("relative")).setPool("pool1").build());
  RemoteIterator iter;
  iter=dfs.listCacheDirectives(null);
  validateListAll(iter,alphaId,alphaId2,betaId,deltaId,relativeId);
  iter=dfs.listCacheDirectives(new CacheDirectiveInfo.Builder().setPool("pool3").build());
  assertFalse(iter.hasNext());
  iter=dfs.listCacheDirectives(new CacheDirectiveInfo.Builder().setPool("pool1").build());
  validateListAll(iter,alphaId,alphaId2,deltaId,relativeId);
  iter=dfs.listCacheDirectives(new CacheDirectiveInfo.Builder().setPool("pool2").build());
  validateListAll(iter,betaId);
  iter=dfs.listCacheDirectives(new CacheDirectiveInfo.Builder().setId(alphaId2).build());
  validateListAll(iter,alphaId2);
  iter=dfs.listCacheDirectives(new CacheDirectiveInfo.Builder().setId(relativeId).build());
  validateListAll(iter,relativeId);
  dfs.removeCacheDirective(betaId);
  iter=dfs.listCacheDirectives(new CacheDirectiveInfo.Builder().setPool("pool2").build());
  assertFalse(iter.hasNext());
  try {
    dfs.removeCacheDirective(betaId);
    fail("expected an error when removing a non-existent ID");
  }
 catch (  InvalidRequestException e) {
    GenericTestUtils.assertExceptionContains("No directive with ID",e);
  }
  try {
    proto.removeCacheDirective(-42l);
    fail("expected an error when removing a negative ID");
  }
 catch (  InvalidRequestException e) {
    GenericTestUtils.assertExceptionContains("Invalid negative ID",e);
  }
  try {
    proto.removeCacheDirective(43l);
    fail("expected an error when removing a non-existent ID");
  }
 catch (  InvalidRequestException e) {
    GenericTestUtils.assertExceptionContains("No directive with ID",e);
  }
  dfs.removeCacheDirective(alphaId);
  dfs.removeCacheDirective(alphaId2);
  dfs.removeCacheDirective(deltaId);
  dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder().setId(relativeId).setReplication((short)555).build());
  iter=dfs.listCacheDirectives(null);
  assertTrue(iter.hasNext());
  CacheDirectiveInfo modified=iter.next().getInfo();
  assertEquals(relativeId,modified.getId().longValue());
  assertEquals((short)555,modified.getReplication().shortValue());
  dfs.removeCacheDirective(relativeId);
  iter=dfs.listCacheDirectives(null);
  assertFalse(iter.hasNext());
  CacheDirectiveInfo directive=new CacheDirectiveInfo.Builder().setPath(new Path(".")).setPool("pool1").build();
  long id=dfs.addCacheDirective(directive);
  dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(directive).setId(id).setReplication((short)2).build());
  dfs.removeCacheDirective(id);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test(timeout=60000) public void testListCachePoolPermissions() throws Exception {
  final UserGroupInformation myUser=UserGroupInformation.createRemoteUser("myuser");
  final DistributedFileSystem myDfs=(DistributedFileSystem)DFSTestUtil.getFileSystemAs(myUser,conf);
  final String poolName="poolparty";
  dfs.addCachePool(new CachePoolInfo(poolName).setMode(new FsPermission((short)0700)));
  RemoteIterator it=myDfs.listCachePools();
  CachePoolInfo info=it.next().getInfo();
  assertFalse(it.hasNext());
  assertEquals("Expected pool name",poolName,info.getPoolName());
  assertNull("Unexpected owner name",info.getOwnerName());
  assertNull("Unexpected group name",info.getGroupName());
  assertNull("Unexpected mode",info.getMode());
  assertNull("Unexpected limit",info.getLimit());
  final long limit=99;
  dfs.modifyCachePool(new CachePoolInfo(poolName).setOwnerName(myUser.getShortUserName()).setLimit(limit));
  it=myDfs.listCachePools();
  info=it.next().getInfo();
  assertFalse(it.hasNext());
  assertEquals("Expected pool name",poolName,info.getPoolName());
  assertEquals("Mismatched owner name",myUser.getShortUserName(),info.getOwnerName());
  assertNotNull("Expected group name",info.getGroupName());
  assertEquals("Mismatched mode",(short)0700,info.getMode().toShort());
  assertEquals("Mismatched limit",limit,(long)info.getLimit());
}

Class: org.apache.hadoop.hdfs.server.namenode.TestCheckPointForSecurityTokens

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests save namespace.
 */
@Test public void testSaveNamespace() throws IOException {
  DistributedFileSystem fs=null;
  try {
    Configuration conf=new HdfsConfiguration();
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,true);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
    cluster.waitActive();
    fs=cluster.getFileSystem();
    FSNamesystem namesystem=cluster.getNamesystem();
    String renewer=UserGroupInformation.getLoginUser().getUserName();
    Token token1=namesystem.getDelegationToken(new Text(renewer));
    Token token2=namesystem.getDelegationToken(new Text(renewer));
    DFSAdmin admin=new DFSAdmin(conf);
    String[] args=new String[]{"-saveNamespace"};
    NameNode nn=cluster.getNameNode();
    for (    StorageDirectory sd : nn.getFSImage().getStorage().dirIterable(null)) {
      EditLogFile log=FSImageTestUtil.findLatestEditsLog(sd);
      assertTrue(log.isInProgress());
      log.validateLog();
      long numTransactions=(log.getLastTxId() - log.getFirstTxId()) + 1;
      assertEquals("In-progress log " + log + " should have 5 transactions",5,numTransactions);
      ;
    }
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    try {
      admin.run(args);
    }
 catch (    Exception e) {
      throw new IOException(e.getMessage());
    }
    for (    StorageDirectory sd : nn.getFSImage().getStorage().dirIterable(null)) {
      EditLogFile log=FSImageTestUtil.findLatestEditsLog(sd);
      assertTrue(log.isInProgress());
      log.validateLog();
      long numTransactions=(log.getLastTxId() - log.getFirstTxId()) + 1;
      assertEquals("In-progress log " + log + " should only have START txn",1,numTransactions);
    }
    cluster.shutdown();
    cluster=null;
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
    cluster.waitActive();
    try {
      renewToken(token1);
      renewToken(token2);
    }
 catch (    IOException e) {
      fail("Could not renew or cancel the token");
    }
    namesystem=cluster.getNamesystem();
    Token token3=namesystem.getDelegationToken(new Text(renewer));
    Token token4=namesystem.getDelegationToken(new Text(renewer));
    cluster.shutdown();
    cluster=null;
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
    cluster.waitActive();
    namesystem=cluster.getNamesystem();
    Token token5=namesystem.getDelegationToken(new Text(renewer));
    try {
      renewToken(token1);
      renewToken(token2);
      renewToken(token3);
      renewToken(token4);
      renewToken(token5);
    }
 catch (    IOException e) {
      fail("Could not renew or cancel the token");
    }
    cluster.shutdown();
    cluster=null;
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
    cluster.waitActive();
    namesystem=cluster.getNamesystem();
    try {
      renewToken(token1);
      cancelToken(token1);
      renewToken(token2);
      cancelToken(token2);
      renewToken(token3);
      cancelToken(token3);
      renewToken(token4);
      cancelToken(token4);
      renewToken(token5);
      cancelToken(token5);
    }
 catch (    IOException e) {
      fail("Could not renew or cancel the token");
    }
  }
  finally {
    if (fs != null)     fs.close();
    if (cluster != null)     cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestCheckpoint

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

@Test(timeout=30000) public void testReloadOnEditReplayFailure() throws IOException {
  Configuration conf=new HdfsConfiguration();
  FSDataOutputStream fos=null;
  SecondaryNameNode secondary=null;
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
    cluster.waitActive();
    fs=cluster.getFileSystem();
    secondary=startSecondaryNameNode(conf);
    fos=fs.create(new Path("tmpfile0"));
    fos.write(new byte[]{0,1,2,3});
    secondary.doCheckpoint();
    fos.write(new byte[]{0,1,2,3});
    fos.hsync();
    Mockito.doThrow(new IOException("Injecting failure during merge")).when(faultInjector).duringMerge();
    try {
      secondary.doCheckpoint();
      fail("Fault injection failed.");
    }
 catch (    IOException ioe) {
    }
    Mockito.reset(faultInjector);
    fos.write(new byte[]{0,1,2,3});
    fos.hsync();
    assertTrue("Another checkpoint should have reloaded image",secondary.doCheckpoint());
  }
  finally {
    if (fs != null) {
      fs.close();
    }
    cleanup(secondary);
    secondary=null;
    cleanup(cluster);
    cluster=null;
    Mockito.reset(faultInjector);
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Tests checkpoint in HDFS.
 */
@Test public void testCheckpoint() throws IOException {
  Path file1=new Path("checkpoint.dat");
  Path file2=new Path("checkpoint2.dat");
  Configuration conf=new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,"0.0.0.0:0");
  replication=(short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY,3);
  MiniDFSCluster cluster=null;
  FileSystem fileSys=null;
  SecondaryNameNode secondary=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
    cluster.waitActive();
    fileSys=cluster.getFileSystem();
    assertTrue(!fileSys.exists(file1));
    assertTrue(!fileSys.exists(file2));
    DFSTestUtil.createFile(fileSys,file1,fileSize,fileSize,blockSize,replication,seed);
    checkFile(fileSys,file1,replication);
    secondary=startSecondaryNameNode(conf);
    secondary.doCheckpoint();
    MetricsRecordBuilder rb=getMetrics(NN_METRICS);
    assertCounterGt("GetImageNumOps",0,rb);
    assertCounterGt("GetEditNumOps",0,rb);
    assertCounterGt("PutImageNumOps",0,rb);
    assertGaugeGt("GetImageAvgTime",0.0,rb);
    assertGaugeGt("GetEditAvgTime",0.0,rb);
    assertGaugeGt("PutImageAvgTime",0.0,rb);
  }
  finally {
    fileSys.close();
    cleanup(secondary);
    secondary=null;
    cleanup(cluster);
    cluster=null;
  }
  Path tmpDir=new Path("/tmp_tmp");
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
    cluster.waitActive();
    fileSys=cluster.getFileSystem();
    checkFile(fileSys,file1,replication);
    cleanupFile(fileSys,file1);
    DFSTestUtil.createFile(fileSys,file2,fileSize,fileSize,blockSize,replication,seed);
    checkFile(fileSys,file2,replication);
    secondary=startSecondaryNameNode(conf);
    secondary.doCheckpoint();
    FSDirectory secondaryFsDir=secondary.getFSNamesystem().dir;
    INode rootInMap=secondaryFsDir.getInode(secondaryFsDir.rootDir.getId());
    assertSame(rootInMap,secondaryFsDir.rootDir);
    fileSys.delete(tmpDir,true);
    fileSys.mkdirs(tmpDir);
    secondary.doCheckpoint();
  }
  finally {
    fileSys.close();
    cleanup(secondary);
    secondary=null;
    cleanup(cluster);
    cluster=null;
  }
  cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
  cluster.waitActive();
  fileSys=cluster.getFileSystem();
  assertTrue(!fileSys.exists(file1));
  assertTrue(fileSys.exists(tmpDir));
  try {
    checkFile(fileSys,file2,replication);
  }
  finally {
    fileSys.close();
    cluster.shutdown();
    cluster=null;
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that the secondary doesn't have to re-download image
 * if it hasn't changed.
 */
@Test public void testSecondaryImageDownload() throws IOException {
  LOG.info("Starting testSecondaryImageDownload");
  Configuration conf=new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY,"0.0.0.0:0");
  Path dir=new Path("/checkpoint");
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(true).build();
  cluster.waitActive();
  FileSystem fileSys=cluster.getFileSystem();
  FSImage image=cluster.getNameNode().getFSImage();
  SecondaryNameNode secondary=null;
  try {
    assertTrue(!fileSys.exists(dir));
    secondary=startSecondaryNameNode(conf);
    File secondaryDir=new File(MiniDFSCluster.getBaseDirectory(),"namesecondary1");
    File secondaryCurrent=new File(secondaryDir,"current");
    long expectedTxIdToDownload=cluster.getNameNode().getFSImage().getStorage().getMostRecentCheckpointTxId();
    File secondaryFsImageBefore=new File(secondaryCurrent,NNStorage.getImageFileName(expectedTxIdToDownload));
    File secondaryFsImageAfter=new File(secondaryCurrent,NNStorage.getImageFileName(expectedTxIdToDownload + 2));
    assertFalse("Secondary should start with empty current/ dir " + "but " + secondaryFsImageBefore + " exists",secondaryFsImageBefore.exists());
    assertTrue("Secondary should have loaded an image",secondary.doCheckpoint());
    assertTrue("Secondary should have downloaded original image",secondaryFsImageBefore.exists());
    assertTrue("Secondary should have created a new image",secondaryFsImageAfter.exists());
    long fsimageLength=secondaryFsImageBefore.length();
    assertEquals("Image size should not have changed",fsimageLength,secondaryFsImageAfter.length());
    fileSys.mkdirs(dir);
    assertFalse("Another checkpoint should not have to re-load image",secondary.doCheckpoint());
    for (    StorageDirectory sd : image.getStorage().dirIterable(NameNodeDirType.IMAGE)) {
      File imageFile=NNStorage.getImageFile(sd,NameNodeFile.IMAGE,expectedTxIdToDownload + 5);
      assertTrue("Image size increased",imageFile.length() > fsimageLength);
    }
  }
  finally {
    fileSys.close();
    cleanup(secondary);
    secondary=null;
    cleanup(cluster);
    cluster=null;
  }
}

APIUtilityVerifier UtilityVerifier 

@Test public void testNameDirError() throws IOException {
  LOG.info("Starting testNameDirError");
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
  Collection nameDirs=cluster.getNameDirs(0);
  cluster.shutdown();
  cluster=null;
  for (  URI nameDirUri : nameDirs) {
    File dir=new File(nameDirUri.getPath());
    try {
      FileUtil.setWritable(dir,false);
      cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).build();
      fail("NN should have failed to start with " + dir + " set unreadable");
    }
 catch (    IOException ioe) {
      GenericTestUtils.assertExceptionContains("storage directory does not exist or is not accessible",ioe);
    }
 finally {
      cleanup(cluster);
      cluster=null;
      FileUtil.setWritable(dir,true);
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Starts two namenodes and two secondary namenodes, verifies that secondary
 * namenodes are configured correctly to talk to their respective namenodes
 * and can do the checkpoint.
 * @throws IOException
 */
@Test public void testMultipleSecondaryNamenodes() throws IOException {
  Configuration conf=new HdfsConfiguration();
  String nameserviceId1="ns1";
  String nameserviceId2="ns2";
  conf.set(DFSConfigKeys.DFS_NAMESERVICES,nameserviceId1 + "," + nameserviceId2);
  MiniDFSCluster cluster=null;
  SecondaryNameNode secondary1=null;
  SecondaryNameNode secondary2=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleFederatedTopology(conf.get(DFSConfigKeys.DFS_NAMESERVICES))).build();
    Configuration snConf1=new HdfsConfiguration(cluster.getConfiguration(0));
    Configuration snConf2=new HdfsConfiguration(cluster.getConfiguration(1));
    InetSocketAddress nn1RpcAddress=cluster.getNameNode(0).getNameNodeAddress();
    InetSocketAddress nn2RpcAddress=cluster.getNameNode(1).getNameNodeAddress();
    String nn1=nn1RpcAddress.getHostName() + ":" + nn1RpcAddress.getPort();
    String nn2=nn2RpcAddress.getHostName() + ":" + nn2RpcAddress.getPort();
    snConf1.set(DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,"");
    snConf2.set(DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,"");
    snConf1.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,nameserviceId1),nn1);
    snConf2.set(DFSUtil.addKeySuffixes(DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,nameserviceId2),nn2);
    secondary1=startSecondaryNameNode(snConf1);
    secondary2=startSecondaryNameNode(snConf2);
    assertEquals(secondary1.getNameNodeAddress().getPort(),nn1RpcAddress.getPort());
    assertEquals(secondary2.getNameNodeAddress().getPort(),nn2RpcAddress.getPort());
    assertTrue(secondary1.getNameNodeAddress().getPort() != secondary2.getNameNodeAddress().getPort());
    secondary1.doCheckpoint();
    secondary2.doCheckpoint();
  }
  finally {
    cleanup(secondary1);
    secondary1=null;
    cleanup(secondary2);
    secondary2=null;
    cleanup(cluster);
    cluster=null;
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testTooManyEditReplayFailures() throws IOException {
  Configuration conf=new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY,"1");
  conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY,"1");
  FSDataOutputStream fos=null;
  SecondaryNameNode secondary=null;
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).checkExitOnShutdown(false).build();
    cluster.waitActive();
    fs=cluster.getFileSystem();
    fos=fs.create(new Path("tmpfile0"));
    fos.write(new byte[]{0,1,2,3});
    Mockito.doThrow(new IOException("Injecting failure during merge")).when(faultInjector).duringMerge();
    secondary=startSecondaryNameNode(conf);
    secondary.doWork();
    fail("2NN did not exit.");
  }
 catch (  ExitException ee) {
    ExitUtil.resetFirstExitException();
    assertEquals("Max retries",1,secondary.getMergeErrorCount() - 1);
  }
 finally {
    if (fs != null) {
      fs.close();
    }
    cleanup(secondary);
    secondary=null;
    cleanup(cluster);
    cluster=null;
    Mockito.reset(faultInjector);
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * Test that a fault while downloading edits does not prevent future
 * checkpointing
 */
@Test(timeout=30000) public void testEditFailureBeforeRename() throws IOException {
  Configuration conf=new HdfsConfiguration();
  SecondaryNameNode secondary=null;
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
    cluster.waitActive();
    fs=cluster.getFileSystem();
    secondary=startSecondaryNameNode(conf);
    DFSTestUtil.createFile(fs,new Path("tmpfile0"),1024,(short)1,0l);
    secondary.doCheckpoint();
    Mockito.doThrow(new IOException("Injecting failure before edit rename")).when(faultInjector).beforeEditsRename();
    DFSTestUtil.createFile(fs,new Path("tmpfile1"),1024,(short)1,0l);
    try {
      secondary.doCheckpoint();
      fail("Fault injection failed.");
    }
 catch (    IOException ioe) {
      GenericTestUtils.assertExceptionContains("Injecting failure before edit rename",ioe);
    }
    Mockito.reset(faultInjector);
    for (    StorageDirectory sd : secondary.getFSImage().getStorage().dirIterable(NameNodeDirType.EDITS)) {
      File[] tmpEdits=sd.getCurrentDir().listFiles(tmpEditsFilter);
      assertTrue("Expected a single tmp edits file in directory " + sd.toString(),tmpEdits.length == 1);
      RandomAccessFile randFile=new RandomAccessFile(tmpEdits[0],"rw");
      randFile.setLength(0);
      randFile.close();
    }
    secondary.doCheckpoint();
  }
  finally {
    if (secondary != null) {
      secondary.shutdown();
    }
    if (fs != null) {
      fs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
    Mockito.reset(faultInjector);
  }
}

APIUtilityVerifier EqualityVerifier 

/** 
 * Regression test for HDFS-3678 "Edit log files are never being purged from 2NN"
 */
@Test public void testSecondaryPurgesEditLogs() throws IOException {
  MiniDFSCluster cluster=null;
  SecondaryNameNode secondary=null;
  Configuration conf=new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY,0);
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).build();
    FileSystem fs=cluster.getFileSystem();
    fs.mkdirs(new Path("/foo"));
    secondary=startSecondaryNameNode(conf);
    for (int i=0; i < 5; i++) {
      secondary.doCheckpoint();
    }
    List checkpointDirs=getCheckpointCurrentDirs(secondary);
    for (    File checkpointDir : checkpointDirs) {
      List editsFiles=FileJournalManager.matchEditLogs(checkpointDir);
      assertEquals("Edit log files were not purged from 2NN",1,editsFiles.size());
    }
  }
  finally {
    cleanup(secondary);
    secondary=null;
    cleanup(cluster);
    cluster=null;
  }
}

APIUtilityVerifier UtilityVerifier BooleanVerifier HybridVerifier 

/** 
 * Test that the primary NN will not serve any files to a 2NN who doesn't
 * share its namespace ID, and also will not accept any files from one.
 */
@Test public void testNamespaceVerifiedOnFileTransfer() throws IOException {
  MiniDFSCluster cluster=null;
  Configuration conf=new HdfsConfiguration();
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).build();
    NamenodeProtocols nn=cluster.getNameNodeRpc();
    URL fsName=DFSUtil.getInfoServer(cluster.getNameNode().getServiceRpcAddress(),conf,DFSUtil.getHttpClientScheme(conf)).toURL();
    nn.rollEditLog();
    RemoteEditLogManifest manifest=nn.getEditLogManifest(1);
    RemoteEditLog log=manifest.getLogs().get(0);
    NNStorage dstImage=Mockito.mock(NNStorage.class);
    Mockito.doReturn(Lists.newArrayList(new File("/wont-be-written"))).when(dstImage).getFiles(Mockito.anyObject(),Mockito.anyString());
    File mockImageFile=File.createTempFile("image","");
    FileOutputStream imageFile=new FileOutputStream(mockImageFile);
    imageFile.write("data".getBytes());
    imageFile.close();
    Mockito.doReturn(mockImageFile).when(dstImage).findImageFile(Mockito.any(NameNodeFile.class),Mockito.anyLong());
    Mockito.doReturn(new StorageInfo(1,1,"X",1,NodeType.NAME_NODE).toColonSeparatedString()).when(dstImage).toColonSeparatedString();
    try {
      TransferFsImage.downloadImageToStorage(fsName,0,dstImage,false);
      fail("Storage info was not verified");
    }
 catch (    IOException ioe) {
      String msg=StringUtils.stringifyException(ioe);
      assertTrue(msg,msg.contains("but the secondary expected"));
    }
    try {
      TransferFsImage.downloadEditsToStorage(fsName,log,dstImage);
      fail("Storage info was not verified");
    }
 catch (    IOException ioe) {
      String msg=StringUtils.stringifyException(ioe);
      assertTrue(msg,msg.contains("but the secondary expected"));
    }
    try {
      TransferFsImage.uploadImageFromStorage(fsName,conf,dstImage,NameNodeFile.IMAGE,0);
      fail("Storage info was not verified");
    }
 catch (    IOException ioe) {
      String msg=StringUtils.stringifyException(ioe);
      assertTrue(msg,msg.contains("but the secondary expected"));
    }
  }
  finally {
    cleanup(cluster);
    cluster=null;
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Tests save namespace.
 */
@Test public void testSaveNamespace() throws IOException {
  MiniDFSCluster cluster=null;
  DistributedFileSystem fs=null;
  FileContext fc;
  try {
    Configuration conf=new HdfsConfiguration();
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(true).build();
    cluster.waitActive();
    fs=(cluster.getFileSystem());
    fc=FileContext.getFileContext(cluster.getURI(0));
    DFSAdmin admin=new DFSAdmin(conf);
    String[] args=new String[]{"-saveNamespace"};
    try {
      admin.run(args);
    }
 catch (    IOException eIO) {
      assertTrue(eIO.getLocalizedMessage().contains("Safe mode should be turned ON"));
    }
catch (    Exception e) {
      throw new IOException(e);
    }
    Path file=new Path("namespace.dat");
    DFSTestUtil.createFile(fs,file,fileSize,fileSize,blockSize,replication,seed);
    checkFile(fs,file,replication);
    Path symlink=new Path("file.link");
    fc.createSymlink(file,symlink,false);
    assertTrue(fc.getFileLinkStatus(symlink).isSymlink());
    Collection editsDirs=cluster.getNameEditsDirs(0);
    for (    URI uri : editsDirs) {
      File ed=new File(uri.getPath());
      assertTrue(new File(ed,"current/" + NNStorage.getInProgressEditsFileName(1)).length() > Integer.SIZE / Byte.SIZE);
    }
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    try {
      admin.run(args);
    }
 catch (    Exception e) {
      throw new IOException(e);
    }
    final int EXPECTED_TXNS_FIRST_SEG=13;
    for (    URI uri : editsDirs) {
      File ed=new File(uri.getPath());
      File curDir=new File(ed,"current");
      LOG.info("Files in " + curDir + ":\n  "+ Joiner.on("\n  ").join(curDir.list()));
      File originalEdits=new File(curDir,NNStorage.getInProgressEditsFileName(1));
      assertFalse(originalEdits.exists());
      File finalizedEdits=new File(curDir,NNStorage.getFinalizedEditsFileName(1,EXPECTED_TXNS_FIRST_SEG));
      GenericTestUtils.assertExists(finalizedEdits);
      assertTrue(finalizedEdits.length() > Integer.SIZE / Byte.SIZE);
      GenericTestUtils.assertExists(new File(ed,"current/" + NNStorage.getInProgressEditsFileName(EXPECTED_TXNS_FIRST_SEG + 1)));
    }
    Collection imageDirs=cluster.getNameDirs(0);
    for (    URI uri : imageDirs) {
      File imageDir=new File(uri.getPath());
      File savedImage=new File(imageDir,"current/" + NNStorage.getImageFileName(EXPECTED_TXNS_FIRST_SEG));
      assertTrue("Should have saved image at " + savedImage,savedImage.exists());
    }
    cluster.shutdown();
    cluster=null;
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(false).build();
    cluster.waitActive();
    fs=(cluster.getFileSystem());
    checkFile(fs,file,replication);
    fc=FileContext.getFileContext(cluster.getURI(0));
    assertTrue(fc.getFileLinkStatus(symlink).isSymlink());
  }
  finally {
    if (fs != null)     fs.close();
    cleanup(cluster);
    cluster=null;
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Regression test for HDFS-3849.  This makes sure that when we re-load the
 * FSImage in the 2NN, we clear the existing leases.
 */
@Test public void testSecondaryNameNodeWithSavedLeases() throws IOException {
  MiniDFSCluster cluster=null;
  SecondaryNameNode secondary=null;
  FSDataOutputStream fos=null;
  Configuration conf=new HdfsConfiguration();
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).format(true).build();
    FileSystem fs=cluster.getFileSystem();
    fos=fs.create(new Path("tmpfile"));
    fos.write(new byte[]{0,1,2,3});
    fos.hflush();
    assertEquals(1,cluster.getNamesystem().getLeaseManager().countLease());
    secondary=startSecondaryNameNode(conf);
    assertEquals(0,secondary.getFSNamesystem().getLeaseManager().countLease());
    secondary.doCheckpoint();
    assertEquals(1,secondary.getFSNamesystem().getLeaseManager().countLease());
    fos.close();
    fos=null;
    cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_ENTER,false);
    cluster.getNameNodeRpc().saveNamespace();
    cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_LEAVE,false);
    secondary.doCheckpoint();
    assertEquals(0,secondary.getFSNamesystem().getLeaseManager().countLease());
  }
  finally {
    if (fos != null) {
      fos.close();
    }
    cleanup(secondary);
    secondary=null;
    cleanup(cluster);
    cluster=null;
  }
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testLegacyOivImage() throws Exception {
  MiniDFSCluster cluster=null;
  SecondaryNameNode secondary=null;
  File tmpDir=Files.createTempDir();
  Configuration conf=new HdfsConfiguration();
  conf.set(DFSConfigKeys.DFS_NAMENODE_LEGACY_OIV_IMAGE_DIR_KEY,tmpDir.getAbsolutePath());
  conf.set(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY,"2");
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).build();
    secondary=startSecondaryNameNode(conf);
    secondary.doCheckpoint();
    String files1[]=tmpDir.list();
    assertEquals("Only one file is expected",1,files1.length);
    secondary.doCheckpoint();
    secondary.doCheckpoint();
    String files2[]=tmpDir.list();
    assertEquals("Two files are expected",2,files2.length);
    for (    String fName : files2) {
      assertFalse(fName.equals(files1[0]));
    }
  }
  finally {
    cleanup(secondary);
    cleanup(cluster);
    tmpDir.delete();
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * Test that the secondary namenode correctly deletes temporary edits
 * on startup.
 */
@Test(timeout=60000) public void testDeleteTemporaryEditsOnStartup() throws IOException {
  Configuration conf=new HdfsConfiguration();
  SecondaryNameNode secondary=null;
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes).build();
    cluster.waitActive();
    fs=cluster.getFileSystem();
    secondary=startSecondaryNameNode(conf);
    DFSTestUtil.createFile(fs,new Path("tmpfile0"),1024,(short)1,0l);
    secondary.doCheckpoint();
    Mockito.doThrow(new IOException("Injecting failure before edit rename")).when(faultInjector).beforeEditsRename();
    DFSTestUtil.createFile(fs,new Path("tmpfile1"),1024,(short)1,0l);
    try {
      secondary.doCheckpoint();
      fail("Fault injection failed.");
    }
 catch (    IOException ioe) {
      GenericTestUtils.assertExceptionContains("Injecting failure before edit rename",ioe);
    }
    Mockito.reset(faultInjector);
    for (    StorageDirectory sd : secondary.getFSImage().getStorage().dirIterable(NameNodeDirType.EDITS)) {
      File[] tmpEdits=sd.getCurrentDir().listFiles(tmpEditsFilter);
      assertTrue("Expected a single tmp edits file in directory " + sd.toString(),tmpEdits.length == 1);
    }
    secondary.shutdown();
    secondary=startSecondaryNameNode(conf);
    for (    StorageDirectory sd : secondary.getFSImage().getStorage().dirIterable(NameNodeDirType.EDITS)) {
      File[] tmpEdits=sd.getCurrentDir().listFiles(tmpEditsFilter);
      assertTrue("Did not expect a tmp edits file in directory " + sd.toString(),tmpEdits.length == 0);
    }
    secondary.doCheckpoint();
  }
  finally {
    if (secondary != null) {
      secondary.shutdown();
    }
    if (fs != null) {
      fs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
    Mockito.reset(faultInjector);
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestClusterId

APIUtilityVerifier BranchVerifier UtilityVerifier EqualityVerifier HybridVerifier 

/** 
 * Test namenode format with -format -force -clusterid option when name
 * directory exists. Format should succeed.
 * @throws IOException
 */
@Test public void testFormatWithForceAndClusterId() throws IOException {
  if (!hdfsDir.mkdirs()) {
    fail("Failed to create dir " + hdfsDir.getPath());
  }
  String myId="testFormatWithForceAndClusterId";
  String[] argv={"-format","-force","-clusterid",myId};
  try {
    NameNode.createNameNode(argv,config);
    fail("createNameNode() did not call System.exit()");
  }
 catch (  ExitException e) {
    assertEquals("Format should have succeeded",0,e.status);
  }
  String cId=getClusterId(config);
  assertEquals("ClusterIds do not match",myId,cId);
}

APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test namenode format with -format -force options when name directory
 * exists. Format should succeed.
 * @throws IOException
 */
@Test public void testFormatWithForce() throws IOException {
  if (!hdfsDir.mkdirs()) {
    fail("Failed to create dir " + hdfsDir.getPath());
  }
  String[] argv={"-format","-force"};
  try {
    NameNode.createNameNode(argv,config);
    fail("createNameNode() did not call System.exit()");
  }
 catch (  ExitException e) {
    assertEquals("Format should have succeeded",0,e.status);
  }
  String cid=getClusterId(config);
  assertTrue("Didn't get new ClusterId",(cid != null && !cid.equals("")));
}

APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test namenode format with -force -nonInteractive -force option. Format
 * should succeed.
 * @throws IOException
 */
@Test public void testFormatWithNonInteractiveAndForce() throws IOException {
  if (!hdfsDir.mkdirs()) {
    fail("Failed to create dir " + hdfsDir.getPath());
  }
  String[] argv={"-format","-nonInteractive","-force"};
  try {
    NameNode.createNameNode(argv,config);
    fail("createNameNode() did not call System.exit()");
  }
 catch (  ExitException e) {
    assertEquals("Format should have succeeded",0,e.status);
  }
  String cid=getClusterId(config);
  assertTrue("Didn't get new ClusterId",(cid != null && !cid.equals("")));
}

APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test namenode format with -format option when an empty name directory
 * exists. Format should succeed.
 * @throws IOException
 */
@Test public void testFormatWithEmptyDir() throws IOException {
  if (!hdfsDir.mkdirs()) {
    fail("Failed to create dir " + hdfsDir.getPath());
  }
  String[] argv={"-format"};
  try {
    NameNode.createNameNode(argv,config);
    fail("createNameNode() did not call System.exit()");
  }
 catch (  ExitException e) {
    assertEquals("Format should have succeeded",0,e.status);
  }
  String cid=getClusterId(config);
  assertTrue("Didn't get new ClusterId",(cid != null && !cid.equals("")));
}

APIUtilityVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test namenode format with -format -nonInteractive options when name
 * directory does not exist. Format should succeed.
 * @throws IOException
 */
@Test public void testFormatWithNonInteractiveNameDirDoesNotExit() throws IOException {
  String[] argv={"-format","-nonInteractive"};
  try {
    NameNode.createNameNode(argv,config);
    fail("createNameNode() did not call System.exit()");
  }
 catch (  ExitException e) {
    assertEquals("Format should have succeeded",0,e.status);
  }
  String cid=getClusterId(config);
  assertTrue("Didn't get new ClusterId",(cid != null && !cid.equals("")));
}

APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test namenode format with -format option when a non empty name directory
 * exists. Enter Y when prompted and the format should succeed.
 * @throws IOException
 * @throws InterruptedException
 */
@Test public void testFormatWithoutForceEnterYes() throws IOException, InterruptedException {
  File data=new File(hdfsDir,"file");
  if (!data.mkdirs()) {
    fail("Failed to create dir " + data.getPath());
  }
  InputStream origIn=System.in;
  ByteArrayInputStream bins=new ByteArrayInputStream("Y\n".getBytes());
  System.setIn(bins);
  String[] argv={"-format"};
  try {
    NameNode.createNameNode(argv,config);
    fail("createNameNode() did not call System.exit()");
  }
 catch (  ExitException e) {
    assertEquals("Format should have succeeded",0,e.status);
  }
  System.setIn(origIn);
  String cid=getClusterId(config);
  assertTrue("Didn't get new ClusterId",(cid != null && !cid.equals("")));
}

APIUtilityVerifier BooleanVerifier 

@Test public void testFormatClusterIdOption() throws IOException {
  NameNode.format(config);
  String cid=getClusterId(config);
  assertTrue("Didn't get new ClusterId",(cid != null && !cid.equals("")));
  StartupOption.FORMAT.setClusterId("mycluster");
  NameNode.format(config);
  cid=getClusterId(config);
  assertTrue("ClusterId didn't match",cid.equals("mycluster"));
  StartupOption.FORMAT.setClusterId("");
  NameNode.format(config);
  String newCid=getClusterId(config);
  assertFalse("ClusterId should not be the same",newCid.equals(cid));
}

APIUtilityVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test namenode format with -format option. Format should succeed.
 * @throws IOException
 */
@Test public void testFormat() throws IOException {
  String[] argv={"-format"};
  try {
    NameNode.createNameNode(argv,config);
    fail("createNameNode() did not call System.exit()");
  }
 catch (  ExitException e) {
    assertEquals("Format should have succeeded",0,e.status);
  }
  String cid=getClusterId(config);
  assertTrue("Didn't get new ClusterId",(cid != null && !cid.equals("")));
}

Class: org.apache.hadoop.hdfs.server.namenode.TestDeadDatanode

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

/** 
 * Test to ensure namenode rejects request from dead datanode
 * - Start a cluster
 * - Shutdown the datanode and wait for it to be marked dead at the namenode
 * - Send datanode requests to Namenode and make sure it is rejected 
 * appropriately.
 */
@Test public void testDeadDatanode() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,500);
  conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,1L);
  cluster=new MiniDFSCluster.Builder(conf).build();
  cluster.waitActive();
  String poolId=cluster.getNamesystem().getBlockPoolId();
  DataNode dn=cluster.getDataNodes().get(0);
  DatanodeRegistration reg=DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().get(0),poolId);
  waitForDatanodeState(reg.getDatanodeUuid(),true,20000);
  dn.shutdown();
  waitForDatanodeState(reg.getDatanodeUuid(),false,20000);
  DatanodeProtocol dnp=cluster.getNameNodeRpc();
  ReceivedDeletedBlockInfo[] blocks={new ReceivedDeletedBlockInfo(new Block(0),ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK,null)};
  StorageReceivedDeletedBlocks[] storageBlocks={new StorageReceivedDeletedBlocks(reg.getDatanodeUuid(),blocks)};
  try {
    dnp.blockReceivedAndDeleted(reg,poolId,storageBlocks);
    fail("Expected IOException is not thrown");
  }
 catch (  IOException ex) {
  }
  StorageBlockReport[] report={new StorageBlockReport(new DatanodeStorage(reg.getDatanodeUuid()),new long[]{0L,0L,0L})};
  try {
    dnp.blockReport(reg,poolId,report);
    fail("Expected IOException is not thrown");
  }
 catch (  IOException ex) {
  }
  StorageReport[] rep={new StorageReport(new DatanodeStorage(reg.getDatanodeUuid()),false,0,0,0,0)};
  DatanodeCommand[] cmd=dnp.sendHeartbeat(reg,rep,0L,0L,0,0,0).getCommands();
  assertEquals(1,cmd.length);
  assertEquals(cmd[0].getAction(),RegisterCommand.REGISTER.getAction());
}

Class: org.apache.hadoop.hdfs.server.namenode.TestEditLog

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test loading an editlog which has had both its storage fail
 * on alternating rolls. Two edit log directories are created.
 * The first one fails on odd rolls, the second on even. Test
 * that we are able to load the entire editlog regardless.
 */
@Test public void testAlternatingJournalFailure() throws IOException {
  File f1=new File(TEST_DIR + "/alternatingjournaltest0");
  File f2=new File(TEST_DIR + "/alternatingjournaltest1");
  List editUris=ImmutableList.of(f1.toURI(),f2.toURI());
  NNStorage storage=setupEdits(editUris,10,new AbortSpec(1,0),new AbortSpec(2,1),new AbortSpec(3,0),new AbortSpec(4,1),new AbortSpec(5,0),new AbortSpec(6,1),new AbortSpec(7,0),new AbortSpec(8,1),new AbortSpec(9,0),new AbortSpec(10,1));
  long totaltxnread=0;
  FSEditLog editlog=getFSEditLog(storage);
  editlog.initJournalsForWrite();
  long startTxId=1;
  Iterable editStreams=editlog.selectInputStreams(startTxId,TXNS_PER_ROLL * 11);
  for (  EditLogInputStream edits : editStreams) {
    FSEditLogLoader.EditLogValidation val=FSEditLogLoader.validateEditLog(edits);
    long read=(val.getEndTxId() - edits.getFirstTxId()) + 1;
    LOG.info("Loading edits " + edits + " read "+ read);
    assertEquals(startTxId,edits.getFirstTxId());
    startTxId+=read;
    totaltxnread+=read;
  }
  editlog.close();
  storage.close();
  assertEquals(TXNS_PER_ROLL * 11,totaltxnread);
}

APIUtilityVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test loading an editlog with gaps. A single editlog directory
 * is set up. On of the edit log files is deleted. This should
 * fail when selecting the input streams as it will not be able 
 * to select enough streams to load up to 4*TXNS_PER_ROLL.
 * There should be 4*TXNS_PER_ROLL transactions as we rolled 3
 * times. 
 */
@Test public void testLoadingWithGaps() throws IOException {
  File f1=new File(TEST_DIR + "/gaptest0");
  List editUris=ImmutableList.of(f1.toURI());
  NNStorage storage=setupEdits(editUris,3);
  final long startGapTxId=1 * TXNS_PER_ROLL + 1;
  final long endGapTxId=2 * TXNS_PER_ROLL;
  File[] files=new File(f1,"current").listFiles(new FilenameFilter(){
    @Override public boolean accept(    File dir,    String name){
      if (name.startsWith(NNStorage.getFinalizedEditsFileName(startGapTxId,endGapTxId))) {
        return true;
      }
      return false;
    }
  }
);
  assertEquals(1,files.length);
  assertTrue(files[0].delete());
  FSEditLog editlog=getFSEditLog(storage);
  editlog.initJournalsForWrite();
  long startTxId=1;
  try {
    editlog.selectInputStreams(startTxId,4 * TXNS_PER_ROLL);
    fail("Should have thrown exception");
  }
 catch (  IOException ioe) {
    GenericTestUtils.assertExceptionContains("Gap in transactions. Expected to be able to read up until " + "at least txid 40 but unable to find any edit logs containing " + "txid 11",ioe);
  }
}

APIUtilityVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test edit log failover.  If a single edit log is missing, other 
 * edits logs should be used instead.
 */
@Test public void testEditLogFailOverFromMissing() throws IOException {
  File f1=new File(TEST_DIR + "/failover0");
  File f2=new File(TEST_DIR + "/failover1");
  List editUris=ImmutableList.of(f1.toURI(),f2.toURI());
  NNStorage storage=setupEdits(editUris,3);
  final long startErrorTxId=1 * TXNS_PER_ROLL + 1;
  final long endErrorTxId=2 * TXNS_PER_ROLL;
  File[] files=new File(f1,"current").listFiles(new FilenameFilter(){
    @Override public boolean accept(    File dir,    String name){
      if (name.startsWith(NNStorage.getFinalizedEditsFileName(startErrorTxId,endErrorTxId))) {
        return true;
      }
      return false;
    }
  }
);
  assertEquals(1,files.length);
  assertTrue(files[0].delete());
  FSEditLog editlog=getFSEditLog(storage);
  editlog.initJournalsForWrite();
  long startTxId=1;
  Collection streams=null;
  try {
    streams=editlog.selectInputStreams(startTxId,4 * TXNS_PER_ROLL);
    readAllEdits(streams,startTxId);
  }
 catch (  IOException e) {
    LOG.error("edit log failover didn't work",e);
    fail("Edit log failover didn't work");
  }
 finally {
    IOUtils.cleanup(null,streams.toArray(new EditLogInputStream[0]));
  }
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

/** 
 * Test edit log failover from a corrupt edit log
 */
@Test public void testEditLogFailOverFromCorrupt() throws IOException {
  File f1=new File(TEST_DIR + "/failover0");
  File f2=new File(TEST_DIR + "/failover1");
  List editUris=ImmutableList.of(f1.toURI(),f2.toURI());
  NNStorage storage=setupEdits(editUris,3);
  final long startErrorTxId=1 * TXNS_PER_ROLL + 1;
  final long endErrorTxId=2 * TXNS_PER_ROLL;
  File[] files=new File(f1,"current").listFiles(new FilenameFilter(){
    @Override public boolean accept(    File dir,    String name){
      if (name.startsWith(NNStorage.getFinalizedEditsFileName(startErrorTxId,endErrorTxId))) {
        return true;
      }
      return false;
    }
  }
);
  assertEquals(1,files.length);
  long fileLen=files[0].length();
  LOG.debug("Corrupting Log File: " + files[0] + " len: "+ fileLen);
  RandomAccessFile rwf=new RandomAccessFile(files[0],"rw");
  rwf.seek(fileLen - 4);
  int b=rwf.readInt();
  rwf.seek(fileLen - 4);
  rwf.writeInt(b + 1);
  rwf.close();
  FSEditLog editlog=getFSEditLog(storage);
  editlog.initJournalsForWrite();
  long startTxId=1;
  Collection streams=null;
  try {
    streams=editlog.selectInputStreams(startTxId,4 * TXNS_PER_ROLL);
    readAllEdits(streams,startTxId);
  }
 catch (  IOException e) {
    LOG.error("edit log failover didn't work",e);
    fail("Edit log failover didn't work");
  }
 finally {
    IOUtils.cleanup(null,streams.toArray(new EditLogInputStream[0]));
  }
}

APIUtilityVerifier EqualityVerifier 

/** 
 * Test case for an empty edit log from a prior version of Hadoop.
 */
@Test public void testPreTxIdEditLogNoEdits() throws Exception {
  FSNamesystem namesys=Mockito.mock(FSNamesystem.class);
  namesys.dir=Mockito.mock(FSDirectory.class);
  long numEdits=testLoad(StringUtils.hexStringToByte("ffffffed"),namesys);
  assertEquals(0,numEdits);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test case for loading a very simple edit log from a format
 * prior to the inclusion of edit transaction IDs in the log.
 */
@Test public void testPreTxidEditLogWithEdits() throws Exception {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    final FSNamesystem namesystem=cluster.getNamesystem();
    long numEdits=testLoad(HADOOP20_SOME_EDITS,namesystem);
    assertEquals(3,numEdits);
    HdfsFileStatus fileInfo=namesystem.getFileInfo("/myfile",false);
    assertEquals("supergroup",fileInfo.getGroup());
    assertEquals(3,fileInfo.getReplication());
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Tests the getEditLogManifest function using mock storage for a number
 * of different situations.
 */
@Test public void testEditLogManifestMocks() throws IOException {
  NNStorage storage;
  FSEditLog log;
  storage=mockStorageWithEdits("[1,100]|[101,200]|[201,]","[1,100]|[101,200]|[201,]");
  log=getFSEditLog(storage);
  log.initJournalsForWrite();
  assertEquals("[[1,100], [101,200]]",log.getEditLogManifest(1).toString());
  assertEquals("[[101,200]]",log.getEditLogManifest(101).toString());
  storage=mockStorageWithEdits("[1,100]|[101,200]","[1,100]|[201,300]|[301,400]");
  log=getFSEditLog(storage);
  log.initJournalsForWrite();
  assertEquals("[[1,100], [101,200], [201,300], [301,400]]",log.getEditLogManifest(1).toString());
  storage=mockStorageWithEdits("[1,100]|[301,400]","[301,400]|[401,500]");
  log=getFSEditLog(storage);
  log.initJournalsForWrite();
  assertEquals("[[301,400], [401,500]]",log.getEditLogManifest(1).toString());
  storage=mockStorageWithEdits("[1,100]|[101,150]","[1,50]|[101,200]");
  log=getFSEditLog(storage);
  log.initJournalsForWrite();
  assertEquals("[[1,100], [101,200]]",log.getEditLogManifest(1).toString());
  assertEquals("[[101,200]]",log.getEditLogManifest(101).toString());
  storage=mockStorageWithEdits("[1,100]|[101,]","[1,100]|[101,200]");
  log=getFSEditLog(storage);
  log.initJournalsForWrite();
  assertEquals("[[1,100], [101,200]]",log.getEditLogManifest(1).toString());
  assertEquals("[[101,200]]",log.getEditLogManifest(101).toString());
}

APIUtilityVerifier UtilityVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testEditChecksum() throws Exception {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  FileSystem fileSys=null;
  cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
  cluster.waitActive();
  fileSys=cluster.getFileSystem();
  final FSNamesystem namesystem=cluster.getNamesystem();
  FSImage fsimage=namesystem.getFSImage();
  final FSEditLog editLog=fsimage.getEditLog();
  fileSys.mkdirs(new Path("/tmp"));
  Iterator iter=fsimage.getStorage().dirIterator(NameNodeDirType.EDITS);
  LinkedList sds=new LinkedList();
  while (iter.hasNext()) {
    sds.add(iter.next());
  }
  editLog.close();
  cluster.shutdown();
  for (  StorageDirectory sd : sds) {
    File editFile=NNStorage.getFinalizedEditsFile(sd,1,3);
    assertTrue(editFile.exists());
    long fileLen=editFile.length();
    LOG.debug("Corrupting Log File: " + editFile + " len: "+ fileLen);
    RandomAccessFile rwf=new RandomAccessFile(editFile,"rw");
    rwf.seek(fileLen - 4);
    int b=rwf.readInt();
    rwf.seek(fileLen - 4);
    rwf.writeInt(b + 1);
    rwf.close();
  }
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).format(false).build();
    fail("should not be able to start");
  }
 catch (  IOException e) {
    assertNotNull("Cause of exception should be ChecksumException",e.getCause());
    assertEquals("Cause of exception should be ChecksumException",ChecksumException.class,e.getCause().getClass());
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestEditLogFileInputStream

APIUtilityVerifier EqualityVerifier ConditionMatcher HybridVerifier 

@Test public void testReadURL() throws Exception {
  HttpURLConnection conn=mock(HttpURLConnection.class);
  doReturn(new ByteArrayInputStream(FAKE_LOG_DATA)).when(conn).getInputStream();
  doReturn(HttpURLConnection.HTTP_OK).when(conn).getResponseCode();
  doReturn(Integer.toString(FAKE_LOG_DATA.length)).when(conn).getHeaderField("Content-Length");
  URLConnectionFactory factory=mock(URLConnectionFactory.class);
  doReturn(conn).when(factory).openConnection(Mockito.any(),anyBoolean());
  URL url=new URL("http://localhost/fakeLog");
  EditLogInputStream elis=EditLogFileInputStream.fromUrl(factory,url,HdfsConstants.INVALID_TXID,HdfsConstants.INVALID_TXID,false);
  EnumMap> counts=FSImageTestUtil.countEditLogOpTypes(elis);
  assertThat(counts.get(FSEditLogOpCodes.OP_ADD).held,is(1));
  assertThat(counts.get(FSEditLogOpCodes.OP_SET_GENSTAMP_V1).held,is(1));
  assertThat(counts.get(FSEditLogOpCodes.OP_CLOSE).held,is(1));
  assertEquals(FAKE_LOG_DATA.length,elis.length());
  elis.close();
}

Class: org.apache.hadoop.hdfs.server.namenode.TestEditLogFileOutputStream

APIUtilityVerifier BooleanVerifier 

/** 
 * Tests EditLogFileOutputStream doesn't throw NullPointerException on
 * close/close sequence. See HDFS-2011.
 */
@Test public void testEditLogFileOutputStreamCloseClose() throws IOException {
  EditLogFileOutputStream editLogStream=new EditLogFileOutputStream(conf,TEST_EDITS,0);
  editLogStream.close();
  try {
    editLogStream.close();
  }
 catch (  IOException ioe) {
    String msg=StringUtils.stringifyException(ioe);
    assertTrue(msg,msg.contains("Trying to use aborted output stream"));
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestEditLogRace

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Tests rolling edit logs while transactions are ongoing.
 */
@Test public void testEditLogRolling() throws Exception {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  FileSystem fileSys=null;
  AtomicReference caughtErr=new AtomicReference();
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
    cluster.waitActive();
    fileSys=cluster.getFileSystem();
    final FSNamesystem namesystem=cluster.getNamesystem();
    FSImage fsimage=namesystem.getFSImage();
    StorageDirectory sd=fsimage.getStorage().getStorageDir(0);
    startTransactionWorkers(namesystem,caughtErr);
    long previousLogTxId=1;
    for (int i=0; i < NUM_ROLLS && caughtErr.get() == null; i++) {
      try {
        Thread.sleep(20);
      }
 catch (      InterruptedException e) {
      }
      LOG.info("Starting roll " + i + ".");
      CheckpointSignature sig=namesystem.rollEditLog();
      long nextLog=sig.curSegmentTxId;
      String logFileName=NNStorage.getFinalizedEditsFileName(previousLogTxId,nextLog - 1);
      previousLogTxId+=verifyEditLogs(namesystem,fsimage,logFileName,previousLogTxId);
      assertEquals(previousLogTxId,nextLog);
      File expectedLog=NNStorage.getInProgressEditsFile(sd,previousLogTxId);
      assertTrue("Expect " + expectedLog + " to exist",expectedLog.exists());
    }
  }
  finally {
    stopTransactionWorkers();
    if (caughtErr.get() != null) {
      throw new RuntimeException(caughtErr.get());
    }
    if (fileSys != null)     fileSys.close();
    if (cluster != null)     cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * The logSync() method in FSEditLog is unsynchronized whiel syncing
 * so that other threads can concurrently enqueue edits while the prior
 * sync is ongoing. This test checks that the log is saved correctly
 * if the saveImage occurs while the syncing thread is in the unsynchronized middle section.
 * This replicates the following manual test proposed by Konstantin:
 * I start the name-node in debugger.
 * I do -mkdir and stop the debugger in logSync() just before it does flush.
 * Then I enter safe mode with another client
 * I start saveNamepsace and stop the debugger in
 * FSImage.saveFSImage() -> FSEditLog.createEditLogFile()
 * -> EditLogFileOutputStream.create() ->
 * after truncating the file but before writing LAYOUT_VERSION into it.
 * Then I let logSync() run.
 * Then I terminate the name-node.
 * After that the name-node wont start, since the edits file is broken.
 */
@Test public void testSaveImageWhileSyncInProgress() throws Exception {
  Configuration conf=getConf();
  NameNode.initMetrics(conf,NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  final FSNamesystem namesystem=FSNamesystem.loadFromDisk(conf);
  try {
    FSImage fsimage=namesystem.getFSImage();
    FSEditLog editLog=fsimage.getEditLog();
    JournalAndStream jas=editLog.getJournals().get(0);
    EditLogFileOutputStream spyElos=spy((EditLogFileOutputStream)jas.getCurrentStream());
    jas.setCurrentStreamForTests(spyElos);
    final AtomicReference deferredException=new AtomicReference();
    final CountDownLatch waitToEnterFlush=new CountDownLatch(1);
    final Thread doAnEditThread=new Thread(){
      @Override public void run(){
        try {
          LOG.info("Starting mkdirs");
          namesystem.mkdirs("/test",new PermissionStatus("test","test",new FsPermission((short)00755)),true);
          LOG.info("mkdirs complete");
        }
 catch (        Throwable ioe) {
          LOG.fatal("Got exception",ioe);
          deferredException.set(ioe);
          waitToEnterFlush.countDown();
        }
      }
    }
;
    Answer blockingFlush=new Answer(){
      @Override public Void answer(      InvocationOnMock invocation) throws Throwable {
        LOG.info("Flush called");
        if (Thread.currentThread() == doAnEditThread) {
          LOG.info("edit thread: Telling main thread we made it to flush section...");
          waitToEnterFlush.countDown();
          LOG.info("edit thread: sleeping for " + BLOCK_TIME + "secs");
          Thread.sleep(BLOCK_TIME * 1000);
          LOG.info("Going through to flush. This will allow the main thread to continue.");
        }
        invocation.callRealMethod();
        LOG.info("Flush complete");
        return null;
      }
    }
;
    doAnswer(blockingFlush).when(spyElos).flush();
    doAnEditThread.start();
    LOG.info("Main thread: waiting to enter flush...");
    waitToEnterFlush.await();
    assertNull(deferredException.get());
    LOG.info("Main thread: detected that logSync is in unsynchronized section.");
    LOG.info("Trying to enter safe mode.");
    LOG.info("This should block for " + BLOCK_TIME + "sec, since flush will sleep that long");
    long st=Time.now();
    namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    long et=Time.now();
    LOG.info("Entered safe mode");
    assertTrue(et - st > (BLOCK_TIME - 1) * 1000);
    namesystem.saveNamespace();
    LOG.info("Joining on edit thread...");
    doAnEditThread.join();
    assertNull(deferredException.get());
    assertEquals(3,verifyEditLogs(namesystem,fsimage,NNStorage.getFinalizedEditsFileName(1,3),1));
    assertEquals(1,verifyEditLogs(namesystem,fsimage,NNStorage.getInProgressEditsFileName(4),4));
  }
  finally {
    LOG.info("Closing namesystem");
    if (namesystem != null)     namesystem.close();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Most of the FSNamesystem methods have a synchronized section where they
 * update the name system itself and write to the edit log, and then
 * unsynchronized, they call logSync. This test verifies that, if an
 * operation has written to the edit log but not yet synced it,
 * we wait for that sync before entering safe mode.
 */
@Test public void testSaveRightBeforeSync() throws Exception {
  Configuration conf=getConf();
  NameNode.initMetrics(conf,NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  final FSNamesystem namesystem=FSNamesystem.loadFromDisk(conf);
  try {
    FSImage fsimage=namesystem.getFSImage();
    FSEditLog editLog=spy(fsimage.getEditLog());
    fsimage.editLog=editLog;
    final AtomicReference deferredException=new AtomicReference();
    final CountDownLatch waitToEnterSync=new CountDownLatch(1);
    final Thread doAnEditThread=new Thread(){
      @Override public void run(){
        try {
          LOG.info("Starting mkdirs");
          namesystem.mkdirs("/test",new PermissionStatus("test","test",new FsPermission((short)00755)),true);
          LOG.info("mkdirs complete");
        }
 catch (        Throwable ioe) {
          LOG.fatal("Got exception",ioe);
          deferredException.set(ioe);
          waitToEnterSync.countDown();
        }
      }
    }
;
    Answer blockingSync=new Answer(){
      @Override public Void answer(      InvocationOnMock invocation) throws Throwable {
        LOG.info("logSync called");
        if (Thread.currentThread() == doAnEditThread) {
          LOG.info("edit thread: Telling main thread we made it just before logSync...");
          waitToEnterSync.countDown();
          LOG.info("edit thread: sleeping for " + BLOCK_TIME + "secs");
          Thread.sleep(BLOCK_TIME * 1000);
          LOG.info("Going through to logSync. This will allow the main thread to continue.");
        }
        invocation.callRealMethod();
        LOG.info("logSync complete");
        return null;
      }
    }
;
    doAnswer(blockingSync).when(editLog).logSync();
    doAnEditThread.start();
    LOG.info("Main thread: waiting to just before logSync...");
    waitToEnterSync.await();
    assertNull(deferredException.get());
    LOG.info("Main thread: detected that logSync about to be called.");
    LOG.info("Trying to enter safe mode.");
    LOG.info("This should block for " + BLOCK_TIME + "sec, since we have pending edits");
    long st=Time.now();
    namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    long et=Time.now();
    LOG.info("Entered safe mode");
    assertTrue(et - st > (BLOCK_TIME - 1) * 1000);
    namesystem.saveNamespace();
    LOG.info("Joining on edit thread...");
    doAnEditThread.join();
    assertNull(deferredException.get());
    assertEquals(3,verifyEditLogs(namesystem,fsimage,NNStorage.getFinalizedEditsFileName(1,3),1));
    assertEquals(1,verifyEditLogs(namesystem,fsimage,NNStorage.getInProgressEditsFileName(4),4));
  }
  finally {
    LOG.info("Closing namesystem");
    if (namesystem != null)     namesystem.close();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestFSDirectory

APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Dump the tree, make some changes, and then dump the tree again. 
 */
@Test public void testDumpTree() throws Exception {
  final INode root=fsdir.getINode("/");
  LOG.info("Original tree");
  final StringBuffer b1=root.dumpTreeRecursively();
  System.out.println("b1=" + b1);
  final BufferedReader in=new BufferedReader(new StringReader(b1.toString()));
  String line=in.readLine();
  checkClassName(line);
  for (; (line=in.readLine()) != null; ) {
    line=line.trim();
    if (!line.isEmpty() && !line.contains("snapshot")) {
      assertTrue("line=" + line,line.startsWith(INodeDirectory.DUMPTREE_LAST_ITEM) || line.startsWith(INodeDirectory.DUMPTREE_EXCEPT_LAST_ITEM));
      checkClassName(line);
    }
  }
}

APIUtilityVerifier IterativeVerifier EqualityVerifier 

/** 
 * Test setting and removing multiple xattrs via single operations
 */
@Test(timeout=300000) public void testXAttrMultiSetRemove() throws Exception {
  List existingXAttrs=Lists.newArrayListWithCapacity(0);
  final Random rand=new Random(0xFEEDA);
  int numExpectedXAttrs=0;
  while (numExpectedXAttrs < numGeneratedXAttrs) {
    LOG.info("Currently have " + numExpectedXAttrs + " xattrs");
    final int numToAdd=rand.nextInt(5) + 1;
    List toAdd=Lists.newArrayListWithCapacity(numToAdd);
    for (int i=0; i < numToAdd; i++) {
      if (numExpectedXAttrs >= numGeneratedXAttrs) {
        break;
      }
      toAdd.add(generatedXAttrs.get(numExpectedXAttrs));
      numExpectedXAttrs++;
    }
    LOG.info("Attempting to add " + toAdd.size() + " XAttrs");
    for (int i=0; i < toAdd.size(); i++) {
      LOG.info("Will add XAttr " + toAdd.get(i));
    }
    List newXAttrs=fsdir.setINodeXAttrs(existingXAttrs,toAdd,EnumSet.of(XAttrSetFlag.CREATE));
    verifyXAttrsPresent(newXAttrs,numExpectedXAttrs);
    existingXAttrs=newXAttrs;
  }
  while (numExpectedXAttrs > 0) {
    LOG.info("Currently have " + numExpectedXAttrs + " xattrs");
    final int numToRemove=rand.nextInt(5) + 1;
    List toRemove=Lists.newArrayListWithCapacity(numToRemove);
    for (int i=0; i < numToRemove; i++) {
      if (numExpectedXAttrs == 0) {
        break;
      }
      toRemove.add(generatedXAttrs.get(numExpectedXAttrs - 1));
      numExpectedXAttrs--;
    }
    final int expectedNumToRemove=toRemove.size();
    LOG.info("Attempting to remove " + expectedNumToRemove + " XAttrs");
    List removedXAttrs=Lists.newArrayList();
    List newXAttrs=fsdir.filterINodeXAttrs(existingXAttrs,toRemove,removedXAttrs);
    assertEquals("Unexpected number of removed XAttrs",expectedNumToRemove,removedXAttrs.size());
    verifyXAttrsPresent(newXAttrs,numExpectedXAttrs);
    existingXAttrs=newXAttrs;
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestFSEditLogLoader

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

@Test public void testDisplayRecentEditLogOpCodes() throws IOException {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  FileSystem fileSys=null;
  cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).enableManagedDfsDirsRedundancy(false).build();
  cluster.waitActive();
  fileSys=cluster.getFileSystem();
  final FSNamesystem namesystem=cluster.getNamesystem();
  FSImage fsimage=namesystem.getFSImage();
  for (int i=0; i < 20; i++) {
    fileSys.mkdirs(new Path("/tmp/tmp" + i));
  }
  StorageDirectory sd=fsimage.getStorage().dirIterator(NameNodeDirType.EDITS).next();
  cluster.shutdown();
  File editFile=FSImageTestUtil.findLatestEditsLog(sd).getFile();
  assertTrue("Should exist: " + editFile,editFile.exists());
  long fileLen=editFile.length();
  RandomAccessFile rwf=new RandomAccessFile(editFile,"rw");
  rwf.seek(fileLen - 40);
  for (int i=0; i < 20; i++) {
    rwf.write(FSEditLogOpCodes.OP_DELETE.getOpCode());
  }
  rwf.close();
  StringBuilder bld=new StringBuilder();
  bld.append("^Error replaying edit log at offset \\d+.  ");
  bld.append("Expected transaction ID was \\d+\n");
  bld.append("Recent opcode offsets: (\\d+\\s*){4}$");
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).enableManagedDfsDirsRedundancy(false).format(false).build();
    fail("should not be able to start");
  }
 catch (  IOException e) {
    assertTrue("error message contains opcodes message",e.getMessage().matches(bld.toString()));
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testValidateEmptyEditLog() throws IOException {
  File testDir=new File(TEST_DIR,"testValidateEmptyEditLog");
  SortedMap offsetToTxId=Maps.newTreeMap();
  File logFile=prepareUnfinalizedTestEditLog(testDir,0,offsetToTxId);
  truncateFile(logFile,8);
  EditLogValidation validation=EditLogFileInputStream.validateEditLog(logFile);
  assertTrue(!validation.hasCorruptHeader());
  assertEquals(HdfsConstants.INVALID_TXID,validation.getEndTxId());
}

APIUtilityVerifier BooleanVerifier 

@Test public void testValidateEditLogWithCorruptHeader() throws IOException {
  File testDir=new File(TEST_DIR,"testValidateEditLogWithCorruptHeader");
  SortedMap offsetToTxId=Maps.newTreeMap();
  File logFile=prepareUnfinalizedTestEditLog(testDir,2,offsetToTxId);
  RandomAccessFile rwf=new RandomAccessFile(logFile,"rw");
  try {
    rwf.seek(0);
    rwf.writeLong(42);
  }
  finally {
    rwf.close();
  }
  EditLogValidation validation=EditLogFileInputStream.validateEditLog(logFile);
  assertTrue(validation.hasCorruptHeader());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testValidateEditLogWithCorruptBody() throws IOException {
  File testDir=new File(TEST_DIR,"testValidateEditLogWithCorruptBody");
  SortedMap offsetToTxId=Maps.newTreeMap();
  final int NUM_TXNS=20;
  File logFile=prepareUnfinalizedTestEditLog(testDir,NUM_TXNS,offsetToTxId);
  File logFileBak=new File(testDir,logFile.getName() + ".bak");
  Files.copy(logFile,logFileBak);
  EditLogValidation validation=EditLogFileInputStream.validateEditLog(logFile);
  assertTrue(!validation.hasCorruptHeader());
  assertEquals(NUM_TXNS + 1,validation.getEndTxId());
  for (  Map.Entry entry : offsetToTxId.entrySet()) {
    long txOffset=entry.getKey();
    long txId=entry.getValue();
    Files.copy(logFileBak,logFile);
    corruptByteInFile(logFile,txOffset);
    validation=EditLogFileInputStream.validateEditLog(logFile);
    long expectedEndTxId=(txId == (NUM_TXNS + 1)) ? NUM_TXNS : (NUM_TXNS + 1);
    assertEquals("Failed when corrupting txn opcode at " + txOffset,expectedEndTxId,validation.getEndTxId());
    assertTrue(!validation.hasCorruptHeader());
  }
  for (  Map.Entry entry : offsetToTxId.entrySet()) {
    long txOffset=entry.getKey();
    long txId=entry.getValue();
    Files.copy(logFileBak,logFile);
    truncateFile(logFile,txOffset);
    validation=EditLogFileInputStream.validateEditLog(logFile);
    long expectedEndTxId=(txId == 0) ? HdfsConstants.INVALID_TXID : (txId - 1);
    assertEquals("Failed when corrupting txid " + txId + " txn opcode "+ "at "+ txOffset,expectedEndTxId,validation.getEndTxId());
    assertTrue(!validation.hasCorruptHeader());
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestFSImage

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Ensure that the digest written by the saver equals to the digest of the
 * file.
 */
@Test public void testDigest() throws IOException {
  Configuration conf=new Configuration();
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    DistributedFileSystem fs=cluster.getFileSystem();
    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fs.saveNamespace();
    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    File currentDir=FSImageTestUtil.getNameNodeCurrentDirs(cluster,0).get(0);
    File fsimage=FSImageTestUtil.findNewestImageFile(currentDir.getAbsolutePath());
    assertEquals(MD5FileUtils.readStoredMd5ForFile(fsimage),MD5FileUtils.computeMd5ForFile(fsimage));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestFSImageWithSnapshot

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Test when there is snapshot taken on root
 */
@Test public void testSnapshotOnRoot() throws Exception {
  final Path root=new Path("/");
  hdfs.allowSnapshot(root);
  hdfs.createSnapshot(root,"s1");
  cluster.shutdown();
  cluster=new MiniDFSCluster.Builder(conf).format(false).numDataNodes(REPLICATION).build();
  cluster.waitActive();
  fsn=cluster.getNamesystem();
  hdfs=cluster.getFileSystem();
  hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
  hdfs.saveNamespace();
  hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  cluster.shutdown();
  cluster=new MiniDFSCluster.Builder(conf).format(false).numDataNodes(REPLICATION).build();
  cluster.waitActive();
  fsn=cluster.getNamesystem();
  hdfs=cluster.getFileSystem();
  INodeDirectory rootNode=fsn.dir.getINode4Write(root.toString()).asDirectory();
  assertTrue("The children list of root should be empty",rootNode.getChildrenList(Snapshot.CURRENT_STATE_ID).isEmpty());
  List diffList=rootNode.getDiffs().asList();
  assertEquals(1,diffList.size());
  Snapshot s1=rootNode.getSnapshot(DFSUtil.string2Bytes("s1"));
  assertEquals(s1.getId(),diffList.get(0).getSnapshotId());
  assertEquals(1,fsn.getSnapshotManager().getNumSnapshottableDirs());
  SnapshottableDirectoryStatus[] sdirs=fsn.getSnapshotManager().getSnapshottableDirListing(null);
  assertEquals(root,sdirs[0].getFullPath());
  hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
  hdfs.saveNamespace();
  hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  cluster.shutdown();
  cluster=new MiniDFSCluster.Builder(conf).format(false).numDataNodes(REPLICATION).build();
  cluster.waitActive();
  fsn=cluster.getNamesystem();
  hdfs=cluster.getFileSystem();
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test fsimage loading when 1) there is an empty file loaded from fsimage,
 * and 2) there is later an append operation to be applied from edit log.
 */
@Test(timeout=60000) public void testLoadImageWithEmptyFile() throws Exception {
  Path file=new Path(dir,"file");
  FSDataOutputStream out=hdfs.create(file);
  out.close();
  hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
  hdfs.saveNamespace();
  hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  out=hdfs.append(file);
  out.write(1);
  out.close();
  cluster.shutdown();
  cluster=new MiniDFSCluster.Builder(conf).format(false).numDataNodes(REPLICATION).build();
  cluster.waitActive();
  hdfs=cluster.getFileSystem();
  FileStatus status=hdfs.getFileStatus(file);
  assertEquals(1,status.getLen());
}

Class: org.apache.hadoop.hdfs.server.namenode.TestFSNamesystem

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testReplQueuesActiveAfterStartupSafemode() throws IOException, InterruptedException {
  Configuration conf=new Configuration();
  FSEditLog fsEditLog=Mockito.mock(FSEditLog.class);
  FSImage fsImage=Mockito.mock(FSImage.class);
  Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog);
  FSNamesystem fsNamesystem=new FSNamesystem(conf,fsImage);
  FSNamesystem fsn=Mockito.spy(fsNamesystem);
  HAContext haContext=Mockito.mock(HAContext.class);
  HAState haState=Mockito.mock(HAState.class);
  Mockito.when(haContext.getState()).thenReturn(haState);
  Mockito.when(haState.shouldPopulateReplQueues()).thenReturn(true);
  Whitebox.setInternalState(fsn,"haContext",haContext);
  NameNode.initMetrics(conf,NamenodeRole.NAMENODE);
  fsn.enterSafeMode(false);
  assertTrue("FSNamesystem didn't enter safemode",fsn.isInSafeMode());
  assertTrue("Replication queues were being populated during very first " + "safemode",!fsn.isPopulatingReplQueues());
  fsn.leaveSafeMode();
  assertTrue("FSNamesystem didn't leave safemode",!fsn.isInSafeMode());
  assertTrue("Replication queues weren't being populated even after leaving " + "safemode",fsn.isPopulatingReplQueues());
  fsn.enterSafeMode(false);
  assertTrue("FSNamesystem didn't enter safemode",fsn.isInSafeMode());
  assertTrue("Replication queues weren't being populated after entering " + "safemode 2nd time",fsn.isPopulatingReplQueues());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test that FSNamesystem#clear clears all leases.
 */
@Test public void testFSNamespaceClearLeases() throws Exception {
  Configuration conf=new HdfsConfiguration();
  File nameDir=new File(MiniDFSCluster.getBaseDirectory(),"name");
  conf.set(DFS_NAMENODE_NAME_DIR_KEY,nameDir.getAbsolutePath());
  NameNode.initMetrics(conf,NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  FSNamesystem fsn=FSNamesystem.loadFromDisk(conf);
  LeaseManager leaseMan=fsn.getLeaseManager();
  leaseMan.addLease("client1","importantFile");
  assertEquals(1,leaseMan.countLease());
  fsn.clear();
  leaseMan=fsn.getLeaseManager();
  assertEquals(0,leaseMan.countLease());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Tests that the namenode edits dirs are gotten with duplicates removed
 */
@Test public void testUniqueEditDirs() throws IOException {
  Configuration config=new Configuration();
  config.set(DFS_NAMENODE_EDITS_DIR_KEY,"file://edits/dir, " + "file://edits/dir1,file://edits/dir1");
  Collection editsDirs=FSNamesystem.getNamespaceEditsDirs(config);
  assertEquals(2,editsDirs.size());
}

Class: org.apache.hadoop.hdfs.server.namenode.TestFSNamesystemMBean

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier HybridVerifier 

@Test public void test() throws Exception {
  Configuration conf=new Configuration();
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).build();
    cluster.waitActive();
    FSNamesystem fsn=cluster.getNameNode().namesystem;
    MBeanServer mbs=ManagementFactory.getPlatformMBeanServer();
    ObjectName mxbeanName=new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState");
    String snapshotStats=(String)(mbs.getAttribute(mxbeanName,"SnapshotStats"));
    @SuppressWarnings("unchecked") Map stat=(Map)JSON.parse(snapshotStats);
    assertTrue(stat.containsKey("SnapshottableDirectories") && (Long)stat.get("SnapshottableDirectories") == fsn.getNumSnapshottableDirs());
    assertTrue(stat.containsKey("Snapshots") && (Long)stat.get("Snapshots") == fsn.getNumSnapshots());
    Object pendingDeletionBlocks=mbs.getAttribute(mxbeanName,"PendingDeletionBlocks");
    assertNotNull(pendingDeletionBlocks);
    assertTrue(pendingDeletionBlocks instanceof Long);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestFavoredNodesEndToEnd

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=180000) public void testFavoredNodesEndToEnd() throws Exception {
  for (int i=0; i < NUM_FILES; i++) {
    Random rand=new Random(System.currentTimeMillis() + i);
    InetSocketAddress datanode[]=getDatanodes(rand);
    Path p=new Path("/filename" + i);
    FSDataOutputStream out=dfs.create(p,FsPermission.getDefault(),true,4096,(short)3,4096L,null,datanode);
    out.write(SOME_BYTES);
    out.close();
    BlockLocation[] locations=getBlockLocations(p);
    for (    BlockLocation loc : locations) {
      String[] hosts=loc.getNames();
      String[] hosts1=getStringForInetSocketAddrs(datanode);
      assertTrue(compareNodes(hosts,hosts1));
    }
  }
}

APIUtilityVerifier IterativeVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

@Test(timeout=180000) public void testWhenSomeNodesAreNotGood() throws Exception {
  final InetSocketAddress addrs[]=new InetSocketAddress[4];
  final String[] hosts=new String[addrs.length];
  for (int i=0; i < addrs.length; i++) {
    addrs[i]=datanodes.get(i).getXferAddress();
    hosts[i]=addrs[i].getAddress().getHostAddress() + ":" + addrs[i].getPort();
  }
  DatanodeInfo d=cluster.getNameNode().getNamesystem().getBlockManager().getDatanodeManager().getDatanodeByXferAddr(addrs[0].getAddress().getHostAddress(),addrs[0].getPort());
  d.setDecommissioned();
  Path p=new Path("/filename-foo-bar-baz");
  final short replication=(short)3;
  FSDataOutputStream out=dfs.create(p,FsPermission.getDefault(),true,4096,replication,4096L,null,addrs);
  out.write(SOME_BYTES);
  out.close();
  d.stopDecommission();
  BlockLocation[] locations=getBlockLocations(p);
  Assert.assertEquals(replication,locations[0].getNames().length);
  ;
  for (int i=0; i < replication; i++) {
    final String loc=locations[0].getNames()[i];
    int j=0;
    for (; j < hosts.length && !loc.equals(hosts[j]); j++)     ;
    Assert.assertTrue("j=" + j,j > 0);
    Assert.assertTrue("loc=" + loc + " not in host list "+ Arrays.asList(hosts)+ ", j="+ j,j < hosts.length);
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestFileJournalManager

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test that inprogress files are handled correct. Set up a single
 * edits directory. Fail on after the last roll. Then verify that the 
 * logs have the expected number of transactions.
 */
@Test public void testInprogressRecovery() throws IOException {
  File f=new File(TestEditLog.TEST_DIR + "/inprogressrecovery");
  NNStorage storage=setupEdits(Collections.singletonList(f.toURI()),5,new AbortSpec(5,0));
  StorageDirectory sd=storage.dirIterator(NameNodeDirType.EDITS).next();
  FileJournalManager jm=new FileJournalManager(conf,sd,storage);
  assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL,getNumberOfTransactions(jm,1,true,false));
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Make sure that in-progress streams aren't counted if we don't ask for
 * them.
 */
@Test public void testExcludeInProgressStreams() throws CorruptionException, IOException {
  File f=new File(TestEditLog.TEST_DIR + "/excludeinprogressstreams");
  NNStorage storage=setupEdits(Collections.singletonList(f.toURI()),10,false);
  StorageDirectory sd=storage.dirIterator(NameNodeDirType.EDITS).next();
  FileJournalManager jm=new FileJournalManager(conf,sd,storage);
  assertEquals(100,getNumberOfTransactions(jm,1,false,false));
  EditLogInputStream elis=getJournalInputStream(jm,90,false);
  try {
    FSEditLogOp lastReadOp=null;
    while ((lastReadOp=elis.readOp()) != null) {
      assertTrue(lastReadOp.getTransactionId() <= 100);
    }
  }
  finally {
    IOUtils.cleanup(LOG,elis);
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that we receive the correct number of transactions when we count
 * the number of transactions around gaps.
 * Set up a single edits directory, with no failures. Delete the 4th logfile.
 * Test that getNumberOfTransactions returns the correct number of 
 * transactions before this gap and after this gap. Also verify that if you
 * try to count on the gap that an exception is thrown.
 */
@Test public void testManyLogsWithGaps() throws IOException {
  File f=new File(TestEditLog.TEST_DIR + "/manylogswithgaps");
  NNStorage storage=setupEdits(Collections.singletonList(f.toURI()),10);
  StorageDirectory sd=storage.dirIterator(NameNodeDirType.EDITS).next();
  final long startGapTxId=3 * TXNS_PER_ROLL + 1;
  final long endGapTxId=4 * TXNS_PER_ROLL;
  File[] files=new File(f,"current").listFiles(new FilenameFilter(){
    @Override public boolean accept(    File dir,    String name){
      if (name.startsWith(NNStorage.getFinalizedEditsFileName(startGapTxId,endGapTxId))) {
        return true;
      }
      return false;
    }
  }
);
  assertEquals(1,files.length);
  assertTrue(files[0].delete());
  FileJournalManager jm=new FileJournalManager(conf,sd,storage);
  assertEquals(startGapTxId - 1,getNumberOfTransactions(jm,1,true,true));
  assertEquals(0,getNumberOfTransactions(jm,startGapTxId,true,true));
  assertEquals(11 * TXNS_PER_ROLL - endGapTxId,getNumberOfTransactions(jm,endGapTxId + 1,true,true));
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test the normal operation of loading transactions from
 * file journal manager. 3 edits directories are setup without any
 * failures. Test that we read in the expected number of transactions.
 */
@Test public void testNormalOperation() throws IOException {
  File f1=new File(TestEditLog.TEST_DIR + "/normtest0");
  File f2=new File(TestEditLog.TEST_DIR + "/normtest1");
  File f3=new File(TestEditLog.TEST_DIR + "/normtest2");
  List editUris=ImmutableList.of(f1.toURI(),f2.toURI(),f3.toURI());
  NNStorage storage=setupEdits(editUris,5);
  long numJournals=0;
  for (  StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
    FileJournalManager jm=new FileJournalManager(conf,sd,storage);
    assertEquals(6 * TXNS_PER_ROLL,getNumberOfTransactions(jm,1,true,false));
    numJournals++;
  }
  assertEquals(3,numJournals);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier ExceptionVerifier HybridVerifier 

@Test(expected=IllegalStateException.class) public void testFinalizeErrorReportedToNNStorage() throws IOException, InterruptedException {
  File f=new File(TestEditLog.TEST_DIR + "/filejournaltestError");
  NNStorage storage=setupEdits(Collections.singletonList(f.toURI()),10,new AbortSpec(10,0));
  StorageDirectory sd=storage.dirIterator(NameNodeDirType.EDITS).next();
  FileJournalManager jm=new FileJournalManager(conf,sd,storage);
  String sdRootPath=sd.getRoot().getAbsolutePath();
  FileUtil.chmod(sdRootPath,"-w",true);
  try {
    jm.finalizeLogSegment(0,1);
  }
  finally {
    FileUtil.chmod(sdRootPath,"+w",true);
    assertTrue(storage.getRemovedStorageDirs().contains(sd));
  }
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Make requests with starting transaction ids which don't match the beginning
 * txid of some log segments.
 * This should succeed.
 */
@Test public void testAskForTransactionsMidfile() throws IOException {
  File f=new File(TestEditLog.TEST_DIR + "/askfortransactionsmidfile");
  NNStorage storage=setupEdits(Collections.singletonList(f.toURI()),10);
  StorageDirectory sd=storage.dirIterator(NameNodeDirType.EDITS).next();
  FileJournalManager jm=new FileJournalManager(conf,sd,storage);
  final int TOTAL_TXIDS=10 * 11;
  for (int txid=1; txid <= TOTAL_TXIDS; txid++) {
    assertEquals((TOTAL_TXIDS - txid) + 1,getNumberOfTransactions(jm,txid,true,false));
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test that we can load an edits directory with a corrupt inprogress file.
 * The corrupt inprogress file should be moved to the side.
 */
@Test public void testManyLogsWithCorruptInprogress() throws IOException {
  File f=new File(TestEditLog.TEST_DIR + "/manylogswithcorruptinprogress");
  NNStorage storage=setupEdits(Collections.singletonList(f.toURI()),10,new AbortSpec(10,0));
  StorageDirectory sd=storage.dirIterator(NameNodeDirType.EDITS).next();
  File[] files=new File(f,"current").listFiles(new FilenameFilter(){
    @Override public boolean accept(    File dir,    String name){
      if (name.startsWith("edits_inprogress")) {
        return true;
      }
      return false;
    }
  }
);
  assertEquals(files.length,1);
  corruptAfterStartSegment(files[0]);
  FileJournalManager jm=new FileJournalManager(conf,sd,storage);
  assertEquals(10 * TXNS_PER_ROLL + 1,getNumberOfTransactions(jm,1,true,false));
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test that FileJournalManager behaves correctly despite inprogress
 * files in all its edit log directories. Set up 3 directories and fail
 * all on the last roll. Verify that the correct number of transaction 
 * are then loaded.
 */
@Test public void testInprogressRecoveryAll() throws IOException {
  File f1=new File(TestEditLog.TEST_DIR + "/failalltest0");
  File f2=new File(TestEditLog.TEST_DIR + "/failalltest1");
  File f3=new File(TestEditLog.TEST_DIR + "/failalltest2");
  List editUris=ImmutableList.of(f1.toURI(),f2.toURI(),f3.toURI());
  NNStorage storage=setupEdits(editUris,5,new AbortSpec(5,0),new AbortSpec(5,1),new AbortSpec(5,2));
  Iterator dirs=storage.dirIterator(NameNodeDirType.EDITS);
  StorageDirectory sd=dirs.next();
  FileJournalManager jm=new FileJournalManager(conf,sd,storage);
  assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL,getNumberOfTransactions(jm,1,true,false));
  sd=dirs.next();
  jm=new FileJournalManager(conf,sd,storage);
  assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL,getNumberOfTransactions(jm,1,true,false));
  sd=dirs.next();
  jm=new FileJournalManager(conf,sd,storage);
  assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL,getNumberOfTransactions(jm,1,true,false));
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test that we can read from a stream created by FileJournalManager.
 * Create a single edits directory, failing it on the final roll.
 * Then try loading from the point of the 3rd roll. Verify that we read 
 * the correct number of transactions from this point.
 */
@Test public void testReadFromStream() throws IOException {
  File f=new File(TestEditLog.TEST_DIR + "/readfromstream");
  NNStorage storage=setupEdits(Collections.singletonList(f.toURI()),10,new AbortSpec(10,0));
  StorageDirectory sd=storage.dirIterator(NameNodeDirType.EDITS).next();
  FileJournalManager jm=new FileJournalManager(conf,sd,storage);
  long expectedTotalTxnCount=TXNS_PER_ROLL * 10 + TXNS_PER_FAIL;
  assertEquals(expectedTotalTxnCount,getNumberOfTransactions(jm,1,true,false));
  long skippedTxns=(3 * TXNS_PER_ROLL);
  long startingTxId=skippedTxns + 1;
  long numLoadable=getNumberOfTransactions(jm,startingTxId,true,false);
  assertEquals(expectedTotalTxnCount - skippedTxns,numLoadable);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Make sure that we starting reading the correct op when we request a stream
 * with a txid in the middle of an edit log file.
 */
@Test public void testReadFromMiddleOfEditLog() throws CorruptionException, IOException {
  File f=new File(TestEditLog.TEST_DIR + "/readfrommiddleofeditlog");
  NNStorage storage=setupEdits(Collections.singletonList(f.toURI()),10);
  StorageDirectory sd=storage.dirIterator(NameNodeDirType.EDITS).next();
  FileJournalManager jm=new FileJournalManager(conf,sd,storage);
  EditLogInputStream elis=getJournalInputStream(jm,5,true);
  try {
    FSEditLogOp op=elis.readOp();
    assertEquals("read unexpected op",op.getTransactionId(),5);
  }
  finally {
    IOUtils.cleanup(LOG,elis);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test a mixture of inprogress files and finalised. Set up 3 edits 
 * directories and fail the second on the last roll. Verify that reading
 * the transactions, reads from the finalised directories.
 */
@Test public void testInprogressRecoveryMixed() throws IOException {
  File f1=new File(TestEditLog.TEST_DIR + "/mixtest0");
  File f2=new File(TestEditLog.TEST_DIR + "/mixtest1");
  File f3=new File(TestEditLog.TEST_DIR + "/mixtest2");
  List editUris=ImmutableList.of(f1.toURI(),f2.toURI(),f3.toURI());
  NNStorage storage=setupEdits(editUris,5,new AbortSpec(5,1));
  Iterator dirs=storage.dirIterator(NameNodeDirType.EDITS);
  StorageDirectory sd=dirs.next();
  FileJournalManager jm=new FileJournalManager(conf,sd,storage);
  assertEquals(6 * TXNS_PER_ROLL,getNumberOfTransactions(jm,1,true,false));
  sd=dirs.next();
  jm=new FileJournalManager(conf,sd,storage);
  assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL,getNumberOfTransactions(jm,1,true,false));
  sd=dirs.next();
  jm=new FileJournalManager(conf,sd,storage);
  assertEquals(6 * TXNS_PER_ROLL,getNumberOfTransactions(jm,1,true,false));
}

Class: org.apache.hadoop.hdfs.server.namenode.TestFsck

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test if fsck can return -1 in case of failure
 * @throws Exception
 */
@Test public void testFsckError() throws Exception {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new HdfsConfiguration();
    cluster=new MiniDFSCluster.Builder(conf).build();
    String fileName="/test.txt";
    Path filePath=new Path(fileName);
    FileSystem fs=cluster.getFileSystem();
    DFSTestUtil.createFile(fs,filePath,1L,(short)1,1L);
    DFSTestUtil.waitReplication(fs,filePath,(short)1);
    INodeFile node=(INodeFile)cluster.getNamesystem().dir.getNode(fileName,true);
    final BlockInfo[] blocks=node.getBlocks();
    assertEquals(blocks.length,1);
    blocks[0].setNumBytes(-1L);
    String outStr=runFsck(conf,-1,true,fileName);
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.FAILURE_STATUS));
    fs.delete(filePath,true);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Test fsck with permission set on inodes 
 */
@Test public void testFsckPermission() throws Exception {
  final DFSTestUtil util=new DFSTestUtil.Builder().setName(getClass().getSimpleName()).setNumFiles(20).build();
  final Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,10000L);
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    final MiniDFSCluster c2=cluster;
    final String dir="/dfsck";
    final Path dirpath=new Path(dir);
    final FileSystem fs=c2.getFileSystem();
    util.createFiles(fs,dir);
    util.waitReplication(fs,dir,(short)3);
    fs.setPermission(dirpath,new FsPermission((short)0700));
    UserGroupInformation fakeUGI=UserGroupInformation.createUserForTesting("ProbablyNotARealUserName",new String[]{"ShangriLa"});
    fakeUGI.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        System.out.println(runFsck(conf,-1,true,dir));
        return null;
      }
    }
);
    fs.setPermission(dirpath,new FsPermission((short)0777));
    fakeUGI.doAs(new PrivilegedExceptionAction(){
      @Override public Object run() throws Exception {
        final String outStr=runFsck(conf,0,true,dir);
        System.out.println(outStr);
        assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
        return null;
      }
    }
);
    util.cleanup(fs,dir);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests that the # of missing block replicas and expected replicas is correct
 * @throws IOException
 */
@Test public void testFsckMissingReplicas() throws IOException {
  final short REPL_FACTOR=2;
  final short NUM_REPLICAS=1;
  final short NUM_BLOCKS=3;
  final long blockSize=512;
  Configuration conf=new Configuration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,blockSize);
  MiniDFSCluster cluster=null;
  DistributedFileSystem dfs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_REPLICAS).build();
    assertNotNull("Failed Cluster Creation",cluster);
    cluster.waitClusterUp();
    dfs=cluster.getFileSystem();
    assertNotNull("Failed to get FileSystem",dfs);
    final String pathString=new String("/testfile");
    final Path path=new Path(pathString);
    long fileLen=blockSize * NUM_BLOCKS;
    DFSTestUtil.createFile(dfs,path,fileLen,REPL_FACTOR,1);
    NameNode namenode=cluster.getNameNode();
    NetworkTopology nettop=cluster.getNamesystem().getBlockManager().getDatanodeManager().getNetworkTopology();
    Map pmap=new HashMap();
    Writer result=new StringWriter();
    PrintWriter out=new PrintWriter(result,true);
    InetAddress remoteAddress=InetAddress.getLocalHost();
    NamenodeFsck fsck=new NamenodeFsck(conf,namenode,nettop,pmap,out,NUM_REPLICAS,(short)1,remoteAddress);
    final HdfsFileStatus file=namenode.getRpcServer().getFileInfo(pathString);
    assertNotNull(file);
    Result res=new Result(conf);
    fsck.check(pathString,file,res);
    System.out.println(result.toString());
    assertEquals(res.missingReplicas,(NUM_BLOCKS * REPL_FACTOR) - (NUM_BLOCKS * NUM_REPLICAS));
    assertEquals(res.numExpectedReplicas,NUM_BLOCKS * REPL_FACTOR);
  }
  finally {
    if (dfs != null) {
      dfs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * do fsck 
 */
@Test public void testFsck() throws Exception {
  DFSTestUtil util=new DFSTestUtil.Builder().setName("TestFsck").setNumFiles(20).build();
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    Configuration conf=new HdfsConfiguration();
    final long precision=1L;
    conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,precision);
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,10000L);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    fs=cluster.getFileSystem();
    final String fileName="/srcdat";
    util.createFiles(fs,fileName);
    util.waitReplication(fs,fileName,(short)3);
    final Path file=new Path(fileName);
    long aTime=fs.getFileStatus(file).getAccessTime();
    Thread.sleep(precision);
    setupAuditLogs();
    String outStr=runFsck(conf,0,true,"/");
    verifyAuditLogs();
    assertEquals(aTime,fs.getFileStatus(file).getAccessTime());
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    if (fs != null) {
      try {
        fs.close();
      }
 catch (      Exception e) {
      }
    }
    cluster.shutdown();
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).build();
    outStr=runFsck(conf,1,true,"/");
    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
    System.out.println(outStr);
    cluster.startDataNodes(conf,4,true,null,null);
    cluster.waitActive();
    cluster.waitClusterUp();
    fs=cluster.getFileSystem();
    util.cleanup(fs,"/srcdat");
  }
  finally {
    if (fs != null) {
      try {
        fs.close();
      }
 catch (      Exception e) {
      }
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testCorruptBlock() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,1000);
  conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE,10);
  FileSystem fs=null;
  DFSClient dfsClient=null;
  LocatedBlocks blocks=null;
  int replicaCount=0;
  Random random=new Random();
  String outStr=null;
  short factor=1;
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    fs=cluster.getFileSystem();
    Path file1=new Path("/testCorruptBlock");
    DFSTestUtil.createFile(fs,file1,1024,factor,0);
    DFSTestUtil.waitReplication(fs,file1,factor);
    ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,file1);
    outStr=runFsck(conf,0,true,"/");
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    File blockFile=MiniDFSCluster.getBlockFile(0,block);
    if (blockFile != null && blockFile.exists()) {
      RandomAccessFile raFile=new RandomAccessFile(blockFile,"rw");
      FileChannel channel=raFile.getChannel();
      String badString="BADBAD";
      int rand=random.nextInt((int)channel.size() / 2);
      raFile.seek(rand);
      raFile.write(badString.getBytes());
      raFile.close();
    }
    try {
      IOUtils.copyBytes(fs.open(file1),new IOUtils.NullOutputStream(),conf,true);
    }
 catch (    IOException ie) {
    }
    dfsClient=new DFSClient(new InetSocketAddress("localhost",cluster.getNameNodePort()),conf);
    blocks=dfsClient.getNamenode().getBlockLocations(file1.toString(),0,Long.MAX_VALUE);
    replicaCount=blocks.get(0).getLocations().length;
    while (replicaCount != factor) {
      try {
        Thread.sleep(100);
      }
 catch (      InterruptedException ignore) {
      }
      blocks=dfsClient.getNamenode().getBlockLocations(file1.toString(),0,Long.MAX_VALUE);
      replicaCount=blocks.get(0).getLocations().length;
    }
    assertTrue(blocks.get(0).isCorrupt());
    outStr=runFsck(conf,1,true,"/");
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
    assertTrue(outStr.contains("testCorruptBlock"));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests that the # of misreplaced replicas is correct
 * @throws IOException
 */
@Test public void testFsckMisPlacedReplicas() throws IOException {
  final short REPL_FACTOR=2;
  short NUM_DN=2;
  final short NUM_BLOCKS=3;
  final long blockSize=512;
  String[] racks={"/rack1","/rack1"};
  String[] hosts={"host1","host2"};
  Configuration conf=new Configuration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,blockSize);
  MiniDFSCluster cluster=null;
  DistributedFileSystem dfs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DN).hosts(hosts).racks(racks).build();
    assertNotNull("Failed Cluster Creation",cluster);
    cluster.waitClusterUp();
    dfs=cluster.getFileSystem();
    assertNotNull("Failed to get FileSystem",dfs);
    final String pathString=new String("/testfile");
    final Path path=new Path(pathString);
    long fileLen=blockSize * NUM_BLOCKS;
    DFSTestUtil.createFile(dfs,path,fileLen,REPL_FACTOR,1);
    NameNode namenode=cluster.getNameNode();
    NetworkTopology nettop=cluster.getNamesystem().getBlockManager().getDatanodeManager().getNetworkTopology();
    nettop.add(DFSTestUtil.getDatanodeDescriptor("/rack2","/host3"));
    NUM_DN++;
    Map pmap=new HashMap();
    Writer result=new StringWriter();
    PrintWriter out=new PrintWriter(result,true);
    InetAddress remoteAddress=InetAddress.getLocalHost();
    NamenodeFsck fsck=new NamenodeFsck(conf,namenode,nettop,pmap,out,NUM_DN,REPL_FACTOR,remoteAddress);
    final HdfsFileStatus file=namenode.getRpcServer().getFileInfo(pathString);
    assertNotNull(file);
    Result res=new Result(conf);
    fsck.check(pathString,file,res);
    assertEquals(res.numMisReplicatedBlocks,NUM_BLOCKS);
  }
  finally {
    if (dfs != null) {
      dfs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier BooleanVerifier 

/** 
 * Test for including the snapshot files in fsck report
 */
@Test public void testFsckForSnapshotFiles() throws Exception {
  final Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    String runFsck=runFsck(conf,0,true,"/","-includeSnapshots","-files");
    assertTrue(runFsck.contains("HEALTHY"));
    final String fileName="/srcdat";
    DistributedFileSystem hdfs=cluster.getFileSystem();
    Path file1=new Path(fileName);
    DFSTestUtil.createFile(hdfs,file1,1024,(short)1,1000L);
    hdfs.allowSnapshot(new Path("/"));
    hdfs.createSnapshot(new Path("/"),"mySnapShot");
    runFsck=runFsck(conf,0,true,"/","-includeSnapshots","-files");
    assertTrue(runFsck.contains("/.snapshot/mySnapShot/srcdat"));
    runFsck=runFsck(conf,0,true,"/","-files");
    assertFalse(runFsck.contains("mySnapShot"));
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier BooleanVerifier 

@Test public void testFsckMoveAndDelete() throws Exception {
  final int MAX_MOVE_TRIES=5;
  DFSTestUtil util=new DFSTestUtil.Builder().setName("TestFsckMoveAndDelete").setNumFiles(5).build();
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    Configuration conf=new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,10000L);
    conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,1);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    String topDir="/srcdat";
    fs=cluster.getFileSystem();
    cluster.waitActive();
    util.createFiles(fs,topDir);
    util.waitReplication(fs,topDir,(short)3);
    String outStr=runFsck(conf,0,true,"/");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    String[] fileNames=util.getFileNames(topDir);
    DFSClient dfsClient=new DFSClient(new InetSocketAddress("localhost",cluster.getNameNodePort()),conf);
    String corruptFileName=fileNames[0];
    ExtendedBlock block=dfsClient.getNamenode().getBlockLocations(corruptFileName,0,Long.MAX_VALUE).get(0).getBlock();
    for (int i=0; i < 4; i++) {
      File blockFile=MiniDFSCluster.getBlockFile(i,block);
      if (blockFile != null && blockFile.exists()) {
        assertTrue(blockFile.delete());
      }
    }
    outStr=runFsck(conf,1,false,"/");
    while (!outStr.contains(NamenodeFsck.CORRUPT_STATUS)) {
      try {
        Thread.sleep(100);
      }
 catch (      InterruptedException ignore) {
      }
      outStr=runFsck(conf,1,false,"/");
    }
    for (int i=0; i < MAX_MOVE_TRIES; i++) {
      outStr=runFsck(conf,1,true,"/","-move");
      assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
      String[] newFileNames=util.getFileNames(topDir);
      boolean found=false;
      for (      String f : newFileNames) {
        if (f.equals(corruptFileName)) {
          found=true;
          break;
        }
      }
      assertTrue(found);
    }
    outStr=runFsck(conf,1,true,"/","-move","-delete");
    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
    outStr=runFsck(conf,0,true,"/");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    util.cleanup(fs,topDir);
    if (fs != null) {
      try {
        fs.close();
      }
 catch (      Exception e) {
      }
    }
    cluster.shutdown();
  }
  finally {
    if (fs != null) {
      try {
        fs.close();
      }
 catch (      Exception e) {
      }
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier IterativeVerifier BranchVerifier InternalCallVerifier BooleanVerifier 

@Test public void testFsckMove() throws Exception {
  Configuration conf=new HdfsConfiguration();
  final int DFS_BLOCK_SIZE=1024;
  final int NUM_DATANODES=4;
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,DFS_BLOCK_SIZE);
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,10000L);
  conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,1);
  DFSTestUtil util=new DFSTestUtil("TestFsck",5,3,(5 * DFS_BLOCK_SIZE) + (DFS_BLOCK_SIZE - 1),5 * DFS_BLOCK_SIZE);
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATANODES).build();
    String topDir="/srcdat";
    fs=cluster.getFileSystem();
    cluster.waitActive();
    util.createFiles(fs,topDir);
    util.waitReplication(fs,topDir,(short)3);
    String outStr=runFsck(conf,0,true,"/");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    DFSClient dfsClient=new DFSClient(new InetSocketAddress("localhost",cluster.getNameNodePort()),conf);
    String fileNames[]=util.getFileNames(topDir);
    CorruptedTestFile ctFiles[]=new CorruptedTestFile[]{new CorruptedTestFile(fileNames[0],Sets.newHashSet(0),dfsClient,NUM_DATANODES,DFS_BLOCK_SIZE),new CorruptedTestFile(fileNames[1],Sets.newHashSet(2,3),dfsClient,NUM_DATANODES,DFS_BLOCK_SIZE),new CorruptedTestFile(fileNames[2],Sets.newHashSet(4),dfsClient,NUM_DATANODES,DFS_BLOCK_SIZE),new CorruptedTestFile(fileNames[3],Sets.newHashSet(0,1,2,3),dfsClient,NUM_DATANODES,DFS_BLOCK_SIZE),new CorruptedTestFile(fileNames[4],Sets.newHashSet(1,2,3,4),dfsClient,NUM_DATANODES,DFS_BLOCK_SIZE)};
    int totalMissingBlocks=0;
    for (    CorruptedTestFile ctFile : ctFiles) {
      totalMissingBlocks+=ctFile.getTotalMissingBlocks();
    }
    for (    CorruptedTestFile ctFile : ctFiles) {
      ctFile.removeBlocks();
    }
    while (true) {
      outStr=runFsck(conf,1,false,"/");
      String numCorrupt=null;
      for (      String line : outStr.split(LINE_SEPARATOR)) {
        Matcher m=numCorruptBlocksPattern.matcher(line);
        if (m.matches()) {
          numCorrupt=m.group(1);
          break;
        }
      }
      if (numCorrupt == null) {
        throw new IOException("failed to find number of corrupt " + "blocks in fsck output.");
      }
      if (numCorrupt.equals(Integer.toString(totalMissingBlocks))) {
        assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
        break;
      }
      try {
        Thread.sleep(100);
      }
 catch (      InterruptedException ignore) {
      }
    }
    outStr=runFsck(conf,1,false,"/","-move");
    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
    for (    CorruptedTestFile ctFile : ctFiles) {
      ctFile.checkSalvagedRemains();
    }
    outStr=runFsck(conf,1,true,"/","-delete");
    assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
    outStr=runFsck(conf,0,true,"/");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    util.cleanup(fs,topDir);
  }
  finally {
    if (fs != null) {
      try {
        fs.close();
      }
 catch (      Exception e) {
      }
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier BooleanVerifier 

/** 
 * Test for checking fsck command on illegal arguments should print the proper
 * usage.
 */
@Test public void testToCheckTheFsckCommandOnIllegalArguments() throws Exception {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new HdfsConfiguration();
    cluster=new MiniDFSCluster.Builder(conf).build();
    String fileName="/test.txt";
    Path filePath=new Path(fileName);
    FileSystem fs=cluster.getFileSystem();
    DFSTestUtil.createFile(fs,filePath,1L,(short)1,1L);
    DFSTestUtil.waitReplication(fs,filePath,(short)1);
    String outStr=runFsck(conf,-1,true,fileName,"-thisIsNotAValidFlag");
    System.out.println(outStr);
    assertTrue(!outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    outStr=runFsck(conf,-1,true,"/",fileName);
    System.out.println(outStr);
    assertTrue(!outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    fs.delete(filePath,true);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testFsckNonExistent() throws Exception {
  DFSTestUtil util=new DFSTestUtil.Builder().setName("TestFsck").setNumFiles(20).build();
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    Configuration conf=new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,10000L);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    fs=cluster.getFileSystem();
    util.createFiles(fs,"/srcdat");
    util.waitReplication(fs,"/srcdat",(short)3);
    String outStr=runFsck(conf,0,true,"/non-existent");
    assertEquals(-1,outStr.indexOf(NamenodeFsck.HEALTHY_STATUS));
    System.out.println(outStr);
    util.cleanup(fs,"/srcdat");
  }
  finally {
    if (fs != null) {
      try {
        fs.close();
      }
 catch (      Exception e) {
      }
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test fsck with symlinks in the filesystem 
 */
@Test public void testFsckSymlink() throws Exception {
  final DFSTestUtil util=new DFSTestUtil.Builder().setName(getClass().getSimpleName()).setNumFiles(1).build();
  final Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,10000L);
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    final long precision=1L;
    conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,precision);
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,10000L);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    fs=cluster.getFileSystem();
    final String fileName="/srcdat";
    util.createFiles(fs,fileName);
    final FileContext fc=FileContext.getFileContext(cluster.getConfiguration(0));
    final Path file=new Path(fileName);
    final Path symlink=new Path("/srcdat-symlink");
    fc.createSymlink(file,symlink,false);
    util.waitReplication(fs,fileName,(short)3);
    long aTime=fc.getFileStatus(symlink).getAccessTime();
    Thread.sleep(precision);
    setupAuditLogs();
    String outStr=runFsck(conf,0,true,"/");
    verifyAuditLogs();
    assertEquals(aTime,fc.getFileStatus(symlink).getAccessTime());
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    assertTrue(outStr.contains("Total symlinks:\t\t1"));
    util.cleanup(fs,fileName);
  }
  finally {
    if (fs != null) {
      try {
        fs.close();
      }
 catch (      Exception e) {
      }
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier 

/** 
 * check if option -list-corruptfiles of fsck command works properly 
 */
@Test public void testFsckListCorruptFilesBlocks() throws Exception {
  Configuration conf=new Configuration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,1000);
  conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,1);
  FileSystem fs=null;
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).build();
    cluster.waitActive();
    fs=cluster.getFileSystem();
    DFSTestUtil util=new DFSTestUtil.Builder().setName("testGetCorruptFiles").setNumFiles(3).setMaxLevels(1).setMaxSize(1024).build();
    util.createFiles(fs,"/corruptData",(short)1);
    util.waitReplication(fs,"/corruptData",(short)1);
    String outStr=runFsck(conf,0,false,"/corruptData","-list-corruptfileblocks");
    System.out.println("1. good fsck out: " + outStr);
    assertTrue(outStr.contains("has 0 CORRUPT files"));
    final String bpid=cluster.getNamesystem().getBlockPoolId();
    for (int i=0; i < 4; i++) {
      for (int j=0; j <= 1; j++) {
        File storageDir=cluster.getInstanceStorageDir(i,j);
        File data_dir=MiniDFSCluster.getFinalizedDir(storageDir,bpid);
        List metadataFiles=MiniDFSCluster.getAllBlockMetadataFiles(data_dir);
        if (metadataFiles == null)         continue;
        for (        File metadataFile : metadataFiles) {
          File blockFile=Block.metaToBlockFile(metadataFile);
          assertTrue("Cannot remove file.",blockFile.delete());
          assertTrue("Cannot remove file.",metadataFile.delete());
        }
      }
    }
    final NamenodeProtocols namenode=cluster.getNameNodeRpc();
    CorruptFileBlocks corruptFileBlocks=namenode.listCorruptFileBlocks("/corruptData",null);
    int numCorrupt=corruptFileBlocks.getFiles().length;
    while (numCorrupt == 0) {
      Thread.sleep(1000);
      corruptFileBlocks=namenode.listCorruptFileBlocks("/corruptData",null);
      numCorrupt=corruptFileBlocks.getFiles().length;
    }
    outStr=runFsck(conf,-1,true,"/corruptData","-list-corruptfileblocks");
    System.out.println("2. bad fsck out: " + outStr);
    assertTrue(outStr.contains("has 3 CORRUPT files"));
    util.createFiles(fs,"/goodData");
    outStr=runFsck(conf,0,true,"/goodData","-list-corruptfileblocks");
    System.out.println("3. good fsck out: " + outStr);
    assertTrue(outStr.contains("has 0 CORRUPT files"));
    util.cleanup(fs,"/corruptData");
    util.cleanup(fs,"/goodData");
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testFsckOpenFiles() throws Exception {
  DFSTestUtil util=new DFSTestUtil.Builder().setName("TestFsck").setNumFiles(4).build();
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    Configuration conf=new HdfsConfiguration();
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,10000L);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
    String topDir="/srcdat";
    String randomString="HADOOP  ";
    fs=cluster.getFileSystem();
    cluster.waitActive();
    util.createFiles(fs,topDir);
    util.waitReplication(fs,topDir,(short)3);
    String outStr=runFsck(conf,0,true,"/");
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    Path openFile=new Path(topDir + "/openFile");
    FSDataOutputStream out=fs.create(openFile);
    int writeCount=0;
    while (writeCount != 100) {
      out.write(randomString.getBytes());
      writeCount++;
    }
    outStr=runFsck(conf,0,true,topDir);
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    assertFalse(outStr.contains("OPENFORWRITE"));
    outStr=runFsck(conf,0,true,topDir,"-openforwrite");
    System.out.println(outStr);
    assertTrue(outStr.contains("OPENFORWRITE"));
    assertTrue(outStr.contains("openFile"));
    out.close();
    outStr=runFsck(conf,0,true,topDir);
    System.out.println(outStr);
    assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
    assertFalse(outStr.contains("OPENFORWRITE"));
    util.cleanup(fs,topDir);
    if (fs != null) {
      try {
        fs.close();
      }
 catch (      Exception e) {
      }
    }
    cluster.shutdown();
  }
  finally {
    if (fs != null) {
      try {
        fs.close();
      }
 catch (      Exception e) {
      }
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestHostsFiles

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testHostsExcludeInUI() throws Exception {
  Configuration conf=getConf();
  short REPLICATION_FACTOR=2;
  final Path filePath=new Path("/testFile");
  FileSystem localFileSys=FileSystem.getLocal(conf);
  Path workingDir=localFileSys.getWorkingDirectory();
  Path dir=new Path(workingDir,"build/test/data/temp/decommission");
  Path excludeFile=new Path(dir,"exclude");
  Path includeFile=new Path(dir,"include");
  assertTrue(localFileSys.mkdirs(dir));
  DFSTestUtil.writeFile(localFileSys,excludeFile,"");
  DFSTestUtil.writeFile(localFileSys,includeFile,"");
  conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE,excludeFile.toUri().getPath());
  conf.set(DFSConfigKeys.DFS_HOSTS,includeFile.toUri().getPath());
  String racks[]={"/rack1","/rack1","/rack2","/rack2"};
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(racks.length).racks(racks).build();
  final FSNamesystem ns=cluster.getNameNode().getNamesystem();
  try {
    final FileSystem fs=cluster.getFileSystem();
    DFSTestUtil.createFile(fs,filePath,1L,REPLICATION_FACTOR,1L);
    ExtendedBlock b=DFSTestUtil.getFirstBlock(fs,filePath);
    DFSTestUtil.waitForReplication(cluster,b,2,REPLICATION_FACTOR,0);
    BlockLocation locs[]=fs.getFileBlockLocations(fs.getFileStatus(filePath),0,Long.MAX_VALUE);
    String name=locs[0].getNames()[0];
    String names=name + "\n" + "localhost:42\n";
    LOG.info("adding '" + names + "' to exclude file "+ excludeFile.toUri().getPath());
    DFSTestUtil.writeFile(localFileSys,excludeFile,name);
    ns.getBlockManager().getDatanodeManager().refreshNodes(conf);
    DFSTestUtil.waitForDecommission(fs,name);
    DFSTestUtil.waitForReplication(cluster,b,2,REPLICATION_FACTOR,0);
    MBeanServer mbs=ManagementFactory.getPlatformMBeanServer();
    ObjectName mxbeanName=new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo");
    String nodes=(String)mbs.getAttribute(mxbeanName,"LiveNodes");
    assertTrue("Live nodes should contain the decommissioned node",nodes.contains("Decommissioned"));
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testHostsIncludeForDeadCount() throws Exception {
  Configuration conf=getConf();
  FileSystem localFileSys=FileSystem.getLocal(conf);
  Path workingDir=localFileSys.getWorkingDirectory();
  Path dir=new Path(workingDir,"build/test/data/temp/decommission");
  Path excludeFile=new Path(dir,"exclude");
  Path includeFile=new Path(dir,"include");
  assertTrue(localFileSys.mkdirs(dir));
  StringBuilder includeHosts=new StringBuilder();
  includeHosts.append("localhost:52").append("\n").append("127.0.0.1:7777").append("\n");
  DFSTestUtil.writeFile(localFileSys,excludeFile,"");
  DFSTestUtil.writeFile(localFileSys,includeFile,includeHosts.toString());
  conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE,excludeFile.toUri().getPath());
  conf.set(DFSConfigKeys.DFS_HOSTS,includeFile.toUri().getPath());
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    final FSNamesystem ns=cluster.getNameNode().getNamesystem();
    assertTrue(ns.getNumDeadDataNodes() == 2);
    assertTrue(ns.getNumLiveDataNodes() == 0);
    MBeanServer mbs=ManagementFactory.getPlatformMBeanServer();
    ObjectName mxbeanName=new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState");
    String nodes=mbs.getAttribute(mxbeanName,"NumDeadDataNodes") + "";
    assertTrue((Integer)mbs.getAttribute(mxbeanName,"NumDeadDataNodes") == 2);
    assertTrue((Integer)mbs.getAttribute(mxbeanName,"NumLiveDataNodes") == 0);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestINodeFile

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

@Test public void testFilesInGetListingOps() throws Exception {
  final Configuration conf=new Configuration();
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    final DistributedFileSystem hdfs=cluster.getFileSystem();
    final FSDirectory fsdir=cluster.getNamesystem().getFSDirectory();
    hdfs.mkdirs(new Path("/tmp"));
    DFSTestUtil.createFile(hdfs,new Path("/tmp/f1"),0,(short)1,0);
    DFSTestUtil.createFile(hdfs,new Path("/tmp/f2"),0,(short)1,0);
    DFSTestUtil.createFile(hdfs,new Path("/tmp/f3"),0,(short)1,0);
    DirectoryListing dl=cluster.getNameNodeRpc().getListing("/tmp",HdfsFileStatus.EMPTY_NAME,false);
    assertTrue(dl.getPartialListing().length == 3);
    String f2=new String("f2");
    dl=cluster.getNameNodeRpc().getListing("/tmp",f2.getBytes(),false);
    assertTrue(dl.getPartialListing().length == 1);
    INode f2INode=fsdir.getINode("/tmp/f2");
    String f2InodePath="/.reserved/.inodes/" + f2INode.getId();
    dl=cluster.getNameNodeRpc().getListing("/tmp",f2InodePath.getBytes(),false);
    assertTrue(dl.getPartialListing().length == 1);
    hdfs.delete(new Path("/tmp/f2"),false);
    try {
      dl=cluster.getNameNodeRpc().getListing("/tmp",f2InodePath.getBytes(),false);
      fail("Didn't get exception for the deleted startAfter token.");
    }
 catch (    IOException e) {
      assertTrue(e instanceof DirectoryListingStartAfterNotFoundException);
    }
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void testPreferredBlockSizeUpperBound(){
  replication=3;
  preferredBlockSize=BLKSIZE_MAXVALUE;
  INodeFile inf=createINodeFile(replication,preferredBlockSize);
  assertEquals("True has to be returned in this case",BLKSIZE_MAXVALUE,inf.getPreferredBlockSize());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests for addressing files using /.reserved/.inodes/ in file system
 * operations.
 */
@Test public void testInodeIdBasedPaths() throws Exception {
  Configuration conf=new Configuration();
  conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,true);
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    DistributedFileSystem fs=cluster.getFileSystem();
    NamenodeProtocols nnRpc=cluster.getNameNodeRpc();
    Path baseDir=getInodePath(INodeId.ROOT_INODE_ID,"testInodeIdBasedPaths");
    Path baseDirRegPath=new Path("/testInodeIdBasedPaths");
    fs.mkdirs(baseDir);
    fs.exists(baseDir);
    long baseDirFileId=nnRpc.getFileInfo(baseDir.toString()).getFileId();
    Path testFileInodePath=getInodePath(baseDirFileId,"test1");
    Path testFileRegularPath=new Path(baseDir,"test1");
    final int testFileBlockSize=1024;
    FileSystemTestHelper.createFile(fs,testFileInodePath,1,testFileBlockSize);
    assertTrue(fs.exists(testFileInodePath));
    FsPermission perm=new FsPermission((short)0666);
    fs.setPermission(testFileInodePath,perm);
    FileStatus fileStatus=fs.getFileStatus(testFileInodePath);
    assertEquals(perm,fileStatus.getPermission());
    fs.setOwner(testFileInodePath,fileStatus.getOwner(),fileStatus.getGroup());
    fs.setTimes(testFileInodePath,0,0);
    fileStatus=fs.getFileStatus(testFileInodePath);
    assertEquals(0,fileStatus.getModificationTime());
    assertEquals(0,fileStatus.getAccessTime());
    fs.setReplication(testFileInodePath,(short)3);
    fileStatus=fs.getFileStatus(testFileInodePath);
    assertEquals(3,fileStatus.getReplication());
    fs.setReplication(testFileInodePath,(short)1);
    assertEquals(testFileBlockSize,nnRpc.getPreferredBlockSize(testFileInodePath.toString()));
{
      fs.isFileClosed(testFileInodePath);
      fs.getAclStatus(testFileInodePath);
      fs.getXAttrs(testFileInodePath);
      fs.listXAttrs(testFileInodePath);
      fs.access(testFileInodePath,FsAction.READ_WRITE);
    }
    String invalidTarget=new Path(baseDir,"invalidTarget").toString();
    String link=new Path(baseDir,"link").toString();
    testInvalidSymlinkTarget(nnRpc,invalidTarget,link);
    String validTarget="/validtarget";
    testValidSymlinkTarget(nnRpc,validTarget,link);
    fs.append(testFileInodePath);
    fs.recoverLease(testFileInodePath);
    LocatedBlocks l1=nnRpc.getBlockLocations(testFileInodePath.toString(),0,Long.MAX_VALUE);
    LocatedBlocks l2=nnRpc.getBlockLocations(testFileRegularPath.toString(),0,Long.MAX_VALUE);
    checkEquals(l1,l2);
    Path renameDst=getInodePath(baseDirFileId,"test2");
    fileStatus=fs.getFileStatus(testFileInodePath);
    fs.rename(testFileInodePath,renameDst);
    fs.rename(renameDst,testFileInodePath);
    assertEquals(fileStatus,fs.getFileStatus(testFileInodePath));
    fs.rename(testFileInodePath,renameDst,Rename.OVERWRITE);
    fs.rename(renameDst,testFileInodePath,Rename.OVERWRITE);
    assertEquals(fileStatus,fs.getFileStatus(testFileInodePath));
    assertEquals(fs.getContentSummary(testFileRegularPath).toString(),fs.getContentSummary(testFileInodePath).toString());
    checkEquals(fs.listFiles(baseDirRegPath,false),fs.listFiles(baseDir,false));
    fs.delete(testFileInodePath,true);
    assertFalse(fs.exists(testFileInodePath));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier EqualityVerifier 

/** 
 * Test for the Replication value. Sets a value and checks if it was set
 * correct.
 */
@Test public void testReplication(){
  replication=3;
  preferredBlockSize=128 * 1024 * 1024;
  INodeFile inf=createINodeFile(replication,preferredBlockSize);
  assertEquals("True has to be returned in this case",replication,inf.getFileReplication());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testDotdotInodePath() throws Exception {
  final Configuration conf=new Configuration();
  MiniDFSCluster cluster=null;
  DFSClient client=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    final DistributedFileSystem hdfs=cluster.getFileSystem();
    final FSDirectory fsdir=cluster.getNamesystem().getFSDirectory();
    final Path dir=new Path("/dir");
    hdfs.mkdirs(dir);
    long dirId=fsdir.getINode(dir.toString()).getId();
    long parentId=fsdir.getINode("/").getId();
    String testPath="/.reserved/.inodes/" + dirId + "/..";
    client=new DFSClient(NameNode.getAddress(conf),conf);
    HdfsFileStatus status=client.getFileInfo(testPath);
    assertTrue(parentId == status.getFileId());
    testPath="/.reserved/.inodes/" + parentId + "/..";
    status=client.getFileInfo(testPath);
    assertTrue(parentId == status.getFileId());
  }
  finally {
    IOUtils.cleanup(LOG,client);
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testXAttrFeature(){
  replication=3;
  preferredBlockSize=128 * 1024 * 1024;
  INodeFile inf=createINodeFile(replication,preferredBlockSize);
  ImmutableList.Builder builder=new ImmutableList.Builder();
  XAttr xAttr=new XAttr.Builder().setNameSpace(XAttr.NameSpace.USER).setName("a1").setValue(new byte[]{0x31,0x32,0x33}).build();
  builder.add(xAttr);
  XAttrFeature f=new XAttrFeature(builder.build());
  inf.addXAttrFeature(f);
  XAttrFeature f1=inf.getXAttrFeature();
  assertEquals(xAttr,f1.getXAttrs().get(0));
  inf.removeXAttrFeature();
  f1=inf.getXAttrFeature();
  assertEquals(f1,null);
}

APIUtilityVerifier UtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests for {@link FSDirectory#resolvePath(String,byte[][],FSDirectory)}
 */
@Test public void testInodePath() throws IOException {
  String path="/a/b/c";
  INode inode=createTreeOfInodes(path);
  FSDirectory fsd=Mockito.mock(FSDirectory.class);
  Mockito.doReturn(inode).when(fsd).getInode(Mockito.anyLong());
  assertEquals("/test",FSDirectory.resolvePath("/test",null,fsd));
  byte[][] components=INode.getPathComponents(path);
  String resolvedPath=FSDirectory.resolvePath(path,components,fsd);
  assertEquals(path,resolvedPath);
  components=INode.getPathComponents("/.reserved/.inodes/1");
  resolvedPath=FSDirectory.resolvePath(path,components,fsd);
  assertEquals(path,resolvedPath);
  components=INode.getPathComponents("/.reserved/.inodes/1/");
  assertEquals(path,resolvedPath);
  components=INode.getPathComponents("/.reserved/.inodes/1/d/e/f");
  resolvedPath=FSDirectory.resolvePath(path,components,fsd);
  assertEquals("/a/b/c/d/e/f",resolvedPath);
  String testPath="/.reserved/.inodes";
  components=INode.getPathComponents(testPath);
  resolvedPath=FSDirectory.resolvePath(testPath,components,fsd);
  assertEquals(testPath,resolvedPath);
  testPath="/.reserved/.inodes/" + INodeId.ROOT_INODE_ID;
  components=INode.getPathComponents(testPath);
  resolvedPath=FSDirectory.resolvePath(testPath,components,fsd);
  assertEquals("/",resolvedPath);
  testPath="/.invalid/.inodes/1";
  components=INode.getPathComponents(testPath);
  resolvedPath=FSDirectory.resolvePath(testPath,components,fsd);
  assertEquals(testPath,resolvedPath);
  Mockito.doReturn(null).when(fsd).getInode(Mockito.anyLong());
  testPath="/.reserved/.inodes/1234";
  components=INode.getPathComponents(testPath);
  try {
    String realPath=FSDirectory.resolvePath(testPath,components,fsd);
    fail("Path should not be resolved:" + realPath);
  }
 catch (  IOException e) {
    assertTrue(e instanceof FileNotFoundException);
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * Test for the static {@link INodeFile#valueOf(INode,String)}and {@link INodeFileUnderConstruction#valueOf(INode,String)} methods.
 * @throws IOException 
 */
@Test public void testValueOf() throws IOException {
  final String path="/testValueOf";
  final short replication=3;
{
    final INode from=null;
    try {
      INodeFile.valueOf(from,path);
      fail();
    }
 catch (    FileNotFoundException fnfe) {
      assertTrue(fnfe.getMessage().contains("File does not exist"));
    }
    try {
      INodeDirectory.valueOf(from,path);
      fail();
    }
 catch (    FileNotFoundException e) {
      assertTrue(e.getMessage().contains("Directory does not exist"));
    }
  }
{
    final INode from=createINodeFile(replication,preferredBlockSize);
    final INodeFile f=INodeFile.valueOf(from,path);
    assertTrue(f == from);
    try {
      INodeDirectory.valueOf(from,path);
      fail();
    }
 catch (    PathIsNotDirectoryException e) {
    }
  }
{
    final INode from=new INodeFile(INodeId.GRANDFATHER_INODE_ID,null,perm,0L,0L,null,replication,1024L);
    from.asFile().toUnderConstruction("client","machine");
    final INodeFile f=INodeFile.valueOf(from,path);
    assertTrue(f == from);
    try {
      INodeDirectory.valueOf(from,path);
      fail();
    }
 catch (    PathIsNotDirectoryException expected) {
    }
  }
{
    final INode from=new INodeDirectory(INodeId.GRANDFATHER_INODE_ID,null,perm,0L);
    try {
      INodeFile.valueOf(from,path);
      fail();
    }
 catch (    FileNotFoundException fnfe) {
      assertTrue(fnfe.getMessage().contains("Path is not a file"));
    }
    final INodeDirectory d=INodeDirectory.valueOf(from,path);
    assertTrue(d == from);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testGetFullPathName(){
  replication=3;
  preferredBlockSize=128 * 1024 * 1024;
  INodeFile inf=createINodeFile(replication,preferredBlockSize);
  inf.setLocalName(DFSUtil.string2Bytes("f"));
  INodeDirectory root=new INodeDirectory(INodeId.GRANDFATHER_INODE_ID,INodeDirectory.ROOT_NAME,perm,0L);
  INodeDirectory dir=new INodeDirectory(INodeId.GRANDFATHER_INODE_ID,DFSUtil.string2Bytes("d"),perm,0L);
  assertEquals("f",inf.getFullPathName());
  dir.addChild(inf);
  assertEquals("d" + Path.SEPARATOR + "f",inf.getFullPathName());
  root.addChild(dir);
  assertEquals(Path.SEPARATOR + "d" + Path.SEPARATOR+ "f",inf.getFullPathName());
  assertEquals(Path.SEPARATOR + "d",dir.getFullPathName());
  assertEquals(Path.SEPARATOR,root.getFullPathName());
}

APIUtilityVerifier EqualityVerifier 

/** 
 * Test for the PreferredBlockSize value. Sets a value and checks if it was
 * set correct.
 */
@Test public void testPreferredBlockSize(){
  replication=3;
  preferredBlockSize=128 * 1024 * 1024;
  INodeFile inf=createINodeFile(replication,preferredBlockSize);
  assertEquals("True has to be returned in this case",preferredBlockSize,inf.getPreferredBlockSize());
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier 

@Test public void testLocationLimitInListingOps() throws Exception {
  final Configuration conf=new Configuration();
  conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT,9);
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    cluster.waitActive();
    final DistributedFileSystem hdfs=cluster.getFileSystem();
    ArrayList source=new ArrayList();
    hdfs.mkdirs(new Path("/tmp1"));
    hdfs.mkdirs(new Path("/tmp2"));
    source.add("f1");
    source.add("f2");
    int numEntries=source.size();
    for (int j=0; j < numEntries; j++) {
      DFSTestUtil.createFile(hdfs,new Path("/tmp1/" + source.get(j)),4096,3 * 1024 - 100,1024,(short)3,0);
    }
    byte[] start=HdfsFileStatus.EMPTY_NAME;
    for (int j=0; j < numEntries; j++) {
      DirectoryListing dl=cluster.getNameNodeRpc().getListing("/tmp1",start,true);
      assertTrue(dl.getPartialListing().length == 1);
      for (int i=0; i < dl.getPartialListing().length; i++) {
        source.remove(dl.getPartialListing()[i].getLocalName());
      }
      start=dl.getLastName();
    }
    assertTrue(source.size() == 0);
    source.add("f1");
    source.add("f2");
    source.add("f3");
    source.add("f4");
    source.add("f5");
    source.add("f6");
    numEntries=source.size();
    for (int j=0; j < numEntries; j++) {
      DFSTestUtil.createFile(hdfs,new Path("/tmp2/" + source.get(j)),4096,3 * 1024 - 100,1024,(short)1,0);
    }
    start=HdfsFileStatus.EMPTY_NAME;
    for (int j=0; j < numEntries / 3; j++) {
      DirectoryListing dl=cluster.getNameNodeRpc().getListing("/tmp2",start,true);
      assertTrue(dl.getPartialListing().length == 3);
      for (int i=0; i < dl.getPartialListing().length; i++) {
        source.remove(dl.getPartialListing()[i].getLocalName());
      }
      start=dl.getLastName();
    }
    assertTrue(source.size() == 0);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier HybridVerifier 

/** 
 * Test whether the inode in inodeMap has been replaced after regular inode
 * replacement
 */
@Test public void testInodeReplacement() throws Exception {
  final Configuration conf=new Configuration();
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    final DistributedFileSystem hdfs=cluster.getFileSystem();
    final FSDirectory fsdir=cluster.getNamesystem().getFSDirectory();
    final Path dir=new Path("/dir");
    hdfs.mkdirs(dir);
    INodeDirectory dirNode=getDir(fsdir,dir);
    INode dirNodeFromNode=fsdir.getInode(dirNode.getId());
    assertSame(dirNode,dirNodeFromNode);
    hdfs.setQuota(dir,Long.MAX_VALUE - 1,Long.MAX_VALUE - 1);
    dirNode=getDir(fsdir,dir);
    assertTrue(dirNode.isWithQuota());
    dirNodeFromNode=fsdir.getInode(dirNode.getId());
    assertSame(dirNode,dirNodeFromNode);
    hdfs.setQuota(dir,-1,-1);
    dirNode=getDir(fsdir,dir);
    dirNodeFromNode=fsdir.getInode(dirNode.getId());
    assertSame(dirNode,dirNodeFromNode);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * FSDirectory#unprotectedSetQuota creates a new INodeDirectoryWithQuota to
 * replace the original INodeDirectory. Before HDFS-4243, the parent field of
 * all the children INodes of the target INodeDirectory is not changed to
 * point to the new INodeDirectoryWithQuota. This testcase tests this
 * scenario.
 */
@Test public void testGetFullPathNameAfterSetQuota() throws Exception {
  long fileLen=1024;
  replication=3;
  Configuration conf=new Configuration();
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(replication).build();
    cluster.waitActive();
    FSNamesystem fsn=cluster.getNamesystem();
    FSDirectory fsdir=fsn.getFSDirectory();
    DistributedFileSystem dfs=cluster.getFileSystem();
    final Path dir=new Path("/dir");
    final Path file=new Path(dir,"file");
    DFSTestUtil.createFile(dfs,file,fileLen,replication,0L);
    INode fnode=fsdir.getINode(file.toString());
    assertEquals(file.toString(),fnode.getFullPathName());
    dfs.setQuota(dir,Long.MAX_VALUE - 1,replication * fileLen * 10);
    INodeDirectory dirNode=getDir(fsdir,dir);
    assertEquals(dir.toString(),dirNode.getFullPathName());
    assertTrue(dirNode.isWithQuota());
    final Path newDir=new Path("/newdir");
    final Path newFile=new Path(newDir,"file");
    dfs.rename(dir,newDir,Options.Rename.OVERWRITE);
    fnode=fsdir.getINode(newFile.toString());
    assertEquals(newFile.toString(),fnode.getFullPathName());
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestListCorruptFileBlocks

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Check that listCorruptFileBlocks works while the namenode is still in safemode.
 */
@Test(timeout=300000) public void testListCorruptFileBlocksInSafeMode() throws Exception {
  MiniDFSCluster cluster=null;
  Random random=new Random();
  try {
    Configuration conf=new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,1);
    conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,3 * 1000);
    conf.setFloat(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_THRESHOLD_PCT_KEY,1.5f);
    conf.setFloat(DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY,0f);
    conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE,10);
    cluster=new MiniDFSCluster.Builder(conf).waitSafeMode(false).build();
    cluster.getNameNodeRpc().setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,false);
    FileSystem fs=cluster.getFileSystem();
    DFSTestUtil util=new DFSTestUtil.Builder().setName("testListCorruptFileBlocksInSafeMode").setNumFiles(2).setMaxLevels(1).setMaxSize(512).build();
    util.createFiles(fs,"/srcdat10");
    Collection badFiles=cluster.getNameNode().getNamesystem().listCorruptFileBlocks("/",null);
    assertTrue("Namenode has " + badFiles.size() + " corrupt files. Expecting None.",badFiles.size() == 0);
    File storageDir=cluster.getInstanceStorageDir(0,0);
    File data_dir=MiniDFSCluster.getFinalizedDir(storageDir,cluster.getNamesystem().getBlockPoolId());
    assertTrue("data directory does not exist",data_dir.exists());
    List metaFiles=MiniDFSCluster.getAllBlockMetadataFiles(data_dir);
    assertTrue("Data directory does not contain any blocks or there was an " + "IO error",metaFiles != null && !metaFiles.isEmpty());
    File metaFile=metaFiles.get(0);
    RandomAccessFile file=new RandomAccessFile(metaFile,"rw");
    FileChannel channel=file.getChannel();
    long position=channel.size() - 2;
    int length=2;
    byte[] buffer=new byte[length];
    random.nextBytes(buffer);
    channel.write(ByteBuffer.wrap(buffer),position);
    file.close();
    LOG.info("Deliberately corrupting file " + metaFile.getName() + " at offset "+ position+ " length "+ length);
    try {
      util.checkFiles(fs,"/srcdat10");
    }
 catch (    BlockMissingException e) {
      System.out.println("Received BlockMissingException as expected.");
    }
catch (    IOException e) {
      assertTrue("Corrupted replicas not handled properly. " + "Expecting BlockMissingException " + " but received IOException "+ e,false);
    }
    badFiles=cluster.getNameNode().getNamesystem().listCorruptFileBlocks("/",null);
    LOG.info("Namenode has bad files. " + badFiles.size());
    assertTrue("Namenode has " + badFiles.size() + " bad files. Expecting 1.",badFiles.size() == 1);
    cluster.restartNameNode(0);
    fs=cluster.getFileSystem();
    while (!cluster.getNameNode().namesystem.isPopulatingReplQueues()) {
      try {
        LOG.info("waiting for replication queues");
        Thread.sleep(1000);
      }
 catch (      InterruptedException ignore) {
      }
    }
    try {
      util.checkFiles(fs,"/srcdat10");
    }
 catch (    BlockMissingException e) {
      System.out.println("Received BlockMissingException as expected.");
    }
catch (    IOException e) {
      assertTrue("Corrupted replicas not handled properly. " + "Expecting BlockMissingException " + " but received IOException "+ e,false);
    }
    badFiles=cluster.getNameNode().getNamesystem().listCorruptFileBlocks("/",null);
    LOG.info("Namenode has bad files. " + badFiles.size());
    assertTrue("Namenode has " + badFiles.size() + " bad files. Expecting 1.",badFiles.size() == 1);
    assertTrue("Namenode is not in safe mode",cluster.getNameNode().isInSafeMode());
    cluster.getNameNodeRpc().setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,false);
    util.cleanup(fs,"/srcdat10");
  }
 catch (  Exception e) {
    LOG.error(StringUtils.stringifyException(e));
    throw e;
  }
 finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Test if NN.listCorruptFiles() returns the right number of results.
 * The corrupt blocks are detected by the BlockPoolSliceScanner.
 * Also, test that DFS.listCorruptFileBlocks can make multiple successive
 * calls.
 */
@Test(timeout=300000) public void testMaxCorruptFiles() throws Exception {
  MiniDFSCluster cluster=null;
  try {
    Configuration conf=new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,3 * 1000);
    cluster=new MiniDFSCluster.Builder(conf).build();
    FileSystem fs=cluster.getFileSystem();
    final int maxCorruptFileBlocks=FSNamesystem.DEFAULT_MAX_CORRUPT_FILEBLOCKS_RETURNED;
    DFSTestUtil util=new DFSTestUtil.Builder().setName("testMaxCorruptFiles").setNumFiles(maxCorruptFileBlocks * 3).setMaxLevels(1).setMaxSize(512).build();
    util.createFiles(fs,"/srcdat2",(short)1);
    util.waitReplication(fs,"/srcdat2",(short)1);
    final NameNode namenode=cluster.getNameNode();
    Collection badFiles=namenode.getNamesystem().listCorruptFileBlocks("/srcdat2",null);
    assertTrue("Namenode has " + badFiles.size() + " corrupt files. Expecting none.",badFiles.size() == 0);
    final String bpid=cluster.getNamesystem().getBlockPoolId();
    for (int i=0; i < 4; i++) {
      for (int j=0; j <= 1; j++) {
        File storageDir=cluster.getInstanceStorageDir(i,j);
        File data_dir=MiniDFSCluster.getFinalizedDir(storageDir,bpid);
        LOG.info("Removing files from " + data_dir);
        List metadataFiles=MiniDFSCluster.getAllBlockMetadataFiles(data_dir);
        if (metadataFiles == null)         continue;
        for (        File metadataFile : metadataFiles) {
          File blockFile=Block.metaToBlockFile(metadataFile);
          assertTrue("Cannot remove file.",blockFile.delete());
          assertTrue("Cannot remove file.",metadataFile.delete());
        }
      }
    }
    LOG.info("Restarting Datanode to trigger BlockPoolSliceScanner");
    cluster.restartDataNodes();
    cluster.waitActive();
    badFiles=namenode.getNamesystem().listCorruptFileBlocks("/srcdat2",null);
    while (badFiles.size() < maxCorruptFileBlocks) {
      LOG.info("# of corrupt files is: " + badFiles.size());
      Thread.sleep(10000);
      badFiles=namenode.getNamesystem().listCorruptFileBlocks("/srcdat2",null);
    }
    badFiles=namenode.getNamesystem().listCorruptFileBlocks("/srcdat2",null);
    LOG.info("Namenode has bad files. " + badFiles.size());
    assertTrue("Namenode has " + badFiles.size() + " bad files. Expecting "+ maxCorruptFileBlocks+ ".",badFiles.size() == maxCorruptFileBlocks);
    CorruptFileBlockIterator iter=(CorruptFileBlockIterator)fs.listCorruptFileBlocks(new Path("/srcdat2"));
    int corruptPaths=countPaths(iter);
    assertTrue("Expected more than " + maxCorruptFileBlocks + " corrupt file blocks but got "+ corruptPaths,corruptPaths > maxCorruptFileBlocks);
    assertTrue("Iterator should have made more than 1 call but made " + iter.getCallsMade(),iter.getCallsMade() > 1);
    util.cleanup(fs,"/srcdat2");
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=300000) public void testlistCorruptFileBlocks() throws Exception {
  Configuration conf=new Configuration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,1000);
  conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,1);
  FileSystem fs=null;
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).build();
    cluster.waitActive();
    fs=cluster.getFileSystem();
    DFSTestUtil util=new DFSTestUtil.Builder().setName("testGetCorruptFiles").setNumFiles(3).setMaxLevels(1).setMaxSize(1024).build();
    util.createFiles(fs,"/corruptData");
    final NameNode namenode=cluster.getNameNode();
    Collection corruptFileBlocks=namenode.getNamesystem().listCorruptFileBlocks("/corruptData",null);
    int numCorrupt=corruptFileBlocks.size();
    assertTrue(numCorrupt == 0);
    String bpid=cluster.getNamesystem().getBlockPoolId();
    for (int i=0; i < 4; i++) {
      for (int j=0; j <= 1; j++) {
        File storageDir=cluster.getInstanceStorageDir(i,j);
        File data_dir=MiniDFSCluster.getFinalizedDir(storageDir,bpid);
        List metadataFiles=MiniDFSCluster.getAllBlockMetadataFiles(data_dir);
        if (metadataFiles == null)         continue;
        for (        File metadataFile : metadataFiles) {
          File blockFile=Block.metaToBlockFile(metadataFile);
          LOG.info("Deliberately removing file " + blockFile.getName());
          assertTrue("Cannot remove file.",blockFile.delete());
          LOG.info("Deliberately removing file " + metadataFile.getName());
          assertTrue("Cannot remove file.",metadataFile.delete());
        }
      }
    }
    int count=0;
    corruptFileBlocks=namenode.getNamesystem().listCorruptFileBlocks("/corruptData",null);
    numCorrupt=corruptFileBlocks.size();
    while (numCorrupt < 3) {
      Thread.sleep(1000);
      corruptFileBlocks=namenode.getNamesystem().listCorruptFileBlocks("/corruptData",null);
      numCorrupt=corruptFileBlocks.size();
      count++;
      if (count > 30)       break;
    }
    LOG.info("Namenode has bad files. " + numCorrupt);
    assertTrue(numCorrupt == 3);
    FSNamesystem.CorruptFileBlockInfo[] cfb=corruptFileBlocks.toArray(new FSNamesystem.CorruptFileBlockInfo[0]);
    String[] cookie=new String[]{"1"};
    Collection nextCorruptFileBlocks=namenode.getNamesystem().listCorruptFileBlocks("/corruptData",cookie);
    FSNamesystem.CorruptFileBlockInfo[] ncfb=nextCorruptFileBlocks.toArray(new FSNamesystem.CorruptFileBlockInfo[0]);
    numCorrupt=nextCorruptFileBlocks.size();
    assertTrue(numCorrupt == 2);
    assertTrue(ncfb[0].block.getBlockName().equalsIgnoreCase(cfb[1].block.getBlockName()));
    corruptFileBlocks=namenode.getNamesystem().listCorruptFileBlocks("/corruptData",cookie);
    numCorrupt=corruptFileBlocks.size();
    assertTrue(numCorrupt == 0);
    util.createFiles(fs,"/goodData");
    corruptFileBlocks=namenode.getNamesystem().listCorruptFileBlocks("/goodData",null);
    numCorrupt=corruptFileBlocks.size();
    assertTrue(numCorrupt == 0);
    util.cleanup(fs,"/corruptData");
    util.cleanup(fs,"/goodData");
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * check if nn.getCorruptFiles() returns a file that has corrupted blocks 
 */
@Test(timeout=300000) public void testListCorruptFilesCorruptedBlock() throws Exception {
  MiniDFSCluster cluster=null;
  Random random=new Random();
  try {
    Configuration conf=new HdfsConfiguration();
    conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,1);
    conf.setInt(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,3 * 1000);
    conf.setInt(DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE,10);
    cluster=new MiniDFSCluster.Builder(conf).build();
    FileSystem fs=cluster.getFileSystem();
    DFSTestUtil util=new DFSTestUtil.Builder().setName("testCorruptFilesCorruptedBlock").setNumFiles(2).setMaxLevels(1).setMaxSize(512).build();
    util.createFiles(fs,"/srcdat10");
    final NameNode namenode=cluster.getNameNode();
    Collection badFiles=namenode.getNamesystem().listCorruptFileBlocks("/",null);
    assertTrue("Namenode has " + badFiles.size() + " corrupt files. Expecting None.",badFiles.size() == 0);
    String bpid=cluster.getNamesystem().getBlockPoolId();
    File storageDir=cluster.getInstanceStorageDir(0,1);
    File data_dir=MiniDFSCluster.getFinalizedDir(storageDir,bpid);
    assertTrue("data directory does not exist",data_dir.exists());
    List metaFiles=MiniDFSCluster.getAllBlockMetadataFiles(data_dir);
    assertTrue("Data directory does not contain any blocks or there was an " + "IO error",metaFiles != null && !metaFiles.isEmpty());
    File metaFile=metaFiles.get(0);
    RandomAccessFile file=new RandomAccessFile(metaFile,"rw");
    FileChannel channel=file.getChannel();
    long position=channel.size() - 2;
    int length=2;
    byte[] buffer=new byte[length];
    random.nextBytes(buffer);
    channel.write(ByteBuffer.wrap(buffer),position);
    file.close();
    LOG.info("Deliberately corrupting file " + metaFile.getName() + " at offset "+ position+ " length "+ length);
    try {
      util.checkFiles(fs,"/srcdat10");
    }
 catch (    BlockMissingException e) {
      System.out.println("Received BlockMissingException as expected.");
    }
catch (    IOException e) {
      assertTrue("Corrupted replicas not handled properly. Expecting BlockMissingException " + " but received IOException " + e,false);
    }
    badFiles=namenode.getNamesystem().listCorruptFileBlocks("/",null);
    LOG.info("Namenode has bad files. " + badFiles.size());
    assertTrue("Namenode has " + badFiles.size() + " bad files. Expecting 1.",badFiles.size() == 1);
    util.cleanup(fs,"/srcdat10");
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier 

/** 
 * test listCorruptFileBlocks in DistributedFileSystem
 */
@Test(timeout=300000) public void testlistCorruptFileBlocksDFS() throws Exception {
  Configuration conf=new Configuration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,1000);
  conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,1);
  FileSystem fs=null;
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).build();
    cluster.waitActive();
    fs=cluster.getFileSystem();
    DistributedFileSystem dfs=(DistributedFileSystem)fs;
    DFSTestUtil util=new DFSTestUtil.Builder().setName("testGetCorruptFiles").setNumFiles(3).setMaxLevels(1).setMaxSize(1024).build();
    util.createFiles(fs,"/corruptData");
    RemoteIterator corruptFileBlocks=dfs.listCorruptFileBlocks(new Path("/corruptData"));
    int numCorrupt=countPaths(corruptFileBlocks);
    assertTrue(numCorrupt == 0);
    String bpid=cluster.getNamesystem().getBlockPoolId();
    for (int i=0; i < 2; i++) {
      File storageDir=cluster.getInstanceStorageDir(0,i);
      File data_dir=MiniDFSCluster.getFinalizedDir(storageDir,bpid);
      List metadataFiles=MiniDFSCluster.getAllBlockMetadataFiles(data_dir);
      if (metadataFiles == null)       continue;
      for (      File metadataFile : metadataFiles) {
        File blockFile=Block.metaToBlockFile(metadataFile);
        LOG.info("Deliberately removing file " + blockFile.getName());
        assertTrue("Cannot remove file.",blockFile.delete());
        LOG.info("Deliberately removing file " + metadataFile.getName());
        assertTrue("Cannot remove file.",metadataFile.delete());
      }
    }
    int count=0;
    corruptFileBlocks=dfs.listCorruptFileBlocks(new Path("/corruptData"));
    numCorrupt=countPaths(corruptFileBlocks);
    while (numCorrupt < 3) {
      Thread.sleep(1000);
      corruptFileBlocks=dfs.listCorruptFileBlocks(new Path("/corruptData"));
      numCorrupt=countPaths(corruptFileBlocks);
      count++;
      if (count > 30)       break;
    }
    LOG.info("Namenode has bad files. " + numCorrupt);
    assertTrue(numCorrupt == 3);
    util.cleanup(fs,"/corruptData");
    util.cleanup(fs,"/goodData");
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestMetaSave

APIUtilityVerifier BooleanVerifier 

/** 
 * Tests metasave after delete, to make sure there are no orphaned blocks
 */
@Test public void testMetasaveAfterDelete() throws IOException, InterruptedException {
  for (int i=0; i < 2; i++) {
    Path file=new Path("/filestatus" + i);
    DFSTestUtil.createFile(fileSys,file,1024,1024,blockSize,(short)2,seed);
  }
  cluster.stopDataNode(1);
  Thread.sleep(15000);
  namesystem.setReplication("/filestatus0",(short)4);
  namesystem.delete("/filestatus0",true);
  namesystem.delete("/filestatus1",true);
  namesystem.metaSave("metasaveAfterDelete.out.txt");
  BufferedReader reader=null;
  try {
    FileInputStream fstream=new FileInputStream(getLogFile("metasaveAfterDelete.out.txt"));
    DataInputStream in=new DataInputStream(fstream);
    reader=new BufferedReader(new InputStreamReader(in));
    reader.readLine();
    String line=reader.readLine();
    assertTrue(line.equals("Live Datanodes: 1"));
    line=reader.readLine();
    assertTrue(line.equals("Dead Datanodes: 1"));
    line=reader.readLine();
    assertTrue(line.equals("Metasave: Blocks waiting for replication: 0"));
    line=reader.readLine();
    assertTrue(line.equals("Mis-replicated blocks that have been postponed:"));
    line=reader.readLine();
    assertTrue(line.equals("Metasave: Blocks being replicated: 0"));
  }
  finally {
    if (reader != null)     reader.close();
  }
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Tests metasave
 */
@Test public void testMetaSave() throws IOException, InterruptedException {
  for (int i=0; i < 2; i++) {
    Path file=new Path("/filestatus" + i);
    DFSTestUtil.createFile(fileSys,file,1024,1024,blockSize,(short)2,seed);
  }
  cluster.stopDataNode(1);
  Thread.sleep(15000);
  namesystem.setReplication("/filestatus0",(short)4);
  namesystem.metaSave("metasave.out.txt");
  FileInputStream fstream=new FileInputStream(getLogFile("metasave.out.txt"));
  DataInputStream in=new DataInputStream(fstream);
  BufferedReader reader=null;
  try {
    reader=new BufferedReader(new InputStreamReader(in));
    String line=reader.readLine();
    Assert.assertEquals("3 files and directories, 2 blocks = 5 total filesystem objects",line);
    line=reader.readLine();
    assertTrue(line.equals("Live Datanodes: 1"));
    line=reader.readLine();
    assertTrue(line.equals("Dead Datanodes: 1"));
    line=reader.readLine();
    line=reader.readLine();
    assertTrue(line.matches("^/filestatus[01]:.*"));
  }
  finally {
    if (reader != null)     reader.close();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestNameNodeMXBean

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@SuppressWarnings({"unchecked"}) @Test public void testNameNodeMXBeanInfo() throws Exception {
  Configuration conf=new Configuration();
  conf.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,NativeIO.POSIX.getCacheManipulator().getMemlockLimit());
  conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,1);
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,1);
  MiniDFSCluster cluster=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
    cluster.waitActive();
    FSNamesystem fsn=cluster.getNameNode().namesystem;
    MBeanServer mbs=ManagementFactory.getPlatformMBeanServer();
    ObjectName mxbeanName=new ObjectName("Hadoop:service=NameNode,name=NameNodeInfo");
    FileSystem localFileSys=FileSystem.getLocal(conf);
    Path workingDir=localFileSys.getWorkingDirectory();
    Path dir=new Path(workingDir,"build/test/data/temp/TestNameNodeMXBean");
    Path includeFile=new Path(dir,"include");
    assertTrue(localFileSys.mkdirs(dir));
    StringBuilder includeHosts=new StringBuilder();
    for (    DataNode dn : cluster.getDataNodes()) {
      includeHosts.append(dn.getDisplayName()).append("\n");
    }
    DFSTestUtil.writeFile(localFileSys,includeFile,includeHosts.toString());
    conf.set(DFSConfigKeys.DFS_HOSTS,includeFile.toUri().getPath());
    fsn.getBlockManager().getDatanodeManager().refreshNodes(conf);
    cluster.stopDataNode(0);
    while (fsn.getNumDatanodesInService() != 2) {
      try {
        Thread.sleep(1000);
      }
 catch (      InterruptedException e) {
      }
    }
    String clusterId=(String)mbs.getAttribute(mxbeanName,"ClusterId");
    assertEquals(fsn.getClusterId(),clusterId);
    String blockpoolId=(String)mbs.getAttribute(mxbeanName,"BlockPoolId");
    assertEquals(fsn.getBlockPoolId(),blockpoolId);
    String version=(String)mbs.getAttribute(mxbeanName,"Version");
    assertEquals(fsn.getVersion(),version);
    assertTrue(version.equals(VersionInfo.getVersion() + ", r" + VersionInfo.getRevision()));
    Long used=(Long)mbs.getAttribute(mxbeanName,"Used");
    assertEquals(fsn.getUsed(),used.longValue());
    Long total=(Long)mbs.getAttribute(mxbeanName,"Total");
    assertEquals(fsn.getTotal(),total.longValue());
    String safemode=(String)mbs.getAttribute(mxbeanName,"Safemode");
    assertEquals(fsn.getSafemode(),safemode);
    Long nondfs=(Long)(mbs.getAttribute(mxbeanName,"NonDfsUsedSpace"));
    assertEquals(fsn.getNonDfsUsedSpace(),nondfs.longValue());
    Float percentremaining=(Float)(mbs.getAttribute(mxbeanName,"PercentRemaining"));
    assertEquals(fsn.getPercentRemaining(),percentremaining.floatValue(),DELTA);
    Long totalblocks=(Long)(mbs.getAttribute(mxbeanName,"TotalBlocks"));
    assertEquals(fsn.getTotalBlocks(),totalblocks.longValue());
    String alivenodeinfo=(String)(mbs.getAttribute(mxbeanName,"LiveNodes"));
    Map> liveNodes=(Map>)JSON.parse(alivenodeinfo);
    assertTrue(liveNodes.size() > 0);
    for (    Map liveNode : liveNodes.values()) {
      assertTrue(liveNode.containsKey("nonDfsUsedSpace"));
      assertTrue(((Long)liveNode.get("nonDfsUsedSpace")) > 0);
      assertTrue(liveNode.containsKey("capacity"));
      assertTrue(((Long)liveNode.get("capacity")) > 0);
      assertTrue(liveNode.containsKey("numBlocks"));
      assertTrue(((Long)liveNode.get("numBlocks")) == 0);
    }
    assertEquals(fsn.getLiveNodes(),alivenodeinfo);
    String deadnodeinfo=(String)(mbs.getAttribute(mxbeanName,"DeadNodes"));
    assertEquals(fsn.getDeadNodes(),deadnodeinfo);
    Map> deadNodes=(Map>)JSON.parse(deadnodeinfo);
    assertTrue(deadNodes.size() > 0);
    for (    Map deadNode : deadNodes.values()) {
      assertTrue(deadNode.containsKey("lastContact"));
      assertTrue(deadNode.containsKey("decommissioned"));
      assertTrue(deadNode.containsKey("xferaddr"));
    }
    String nodeUsage=(String)(mbs.getAttribute(mxbeanName,"NodeUsage"));
    assertEquals("Bad value for NodeUsage",fsn.getNodeUsage(),nodeUsage);
    String nameJournalStatus=(String)(mbs.getAttribute(mxbeanName,"NameJournalStatus"));
    assertEquals("Bad value for NameJournalStatus",fsn.getNameJournalStatus(),nameJournalStatus);
    String journalTxnInfo=(String)mbs.getAttribute(mxbeanName,"JournalTransactionInfo");
    assertEquals("Bad value for NameTxnIds",fsn.getJournalTransactionInfo(),journalTxnInfo);
    String nnStarted=(String)mbs.getAttribute(mxbeanName,"NNStarted");
    assertEquals("Bad value for NNStarted",fsn.getNNStarted(),nnStarted);
    String compileInfo=(String)mbs.getAttribute(mxbeanName,"CompileInfo");
    assertEquals("Bad value for CompileInfo",fsn.getCompileInfo(),compileInfo);
    String corruptFiles=(String)(mbs.getAttribute(mxbeanName,"CorruptFiles"));
    assertEquals("Bad value for CorruptFiles",fsn.getCorruptFiles(),corruptFiles);
    String nameDirStatuses=(String)(mbs.getAttribute(mxbeanName,"NameDirStatuses"));
    assertEquals(fsn.getNameDirStatuses(),nameDirStatuses);
    Map> statusMap=(Map>)JSON.parse(nameDirStatuses);
    Collection nameDirUris=cluster.getNameDirs(0);
    for (    URI nameDirUri : nameDirUris) {
      File nameDir=new File(nameDirUri);
      System.out.println("Checking for the presence of " + nameDir + " in active name dirs.");
      assertTrue(statusMap.get("active").containsKey(nameDir.getAbsolutePath()));
    }
    assertEquals(2,statusMap.get("active").size());
    assertEquals(0,statusMap.get("failed").size());
    File failedNameDir=new File(nameDirUris.iterator().next());
    assertEquals(0,FileUtil.chmod(new File(failedNameDir,"current").getAbsolutePath(),"000"));
    cluster.getNameNodeRpc().rollEditLog();
    nameDirStatuses=(String)(mbs.getAttribute(mxbeanName,"NameDirStatuses"));
    statusMap=(Map>)JSON.parse(nameDirStatuses);
    for (    URI nameDirUri : nameDirUris) {
      File nameDir=new File(nameDirUri);
      String expectedStatus=nameDir.equals(failedNameDir) ? "failed" : "active";
      System.out.println("Checking for the presence of " + nameDir + " in "+ expectedStatus+ " name dirs.");
      assertTrue(statusMap.get(expectedStatus).containsKey(nameDir.getAbsolutePath()));
    }
    assertEquals(1,statusMap.get("active").size());
    assertEquals(1,statusMap.get("failed").size());
    assertEquals(0L,mbs.getAttribute(mxbeanName,"CacheUsed"));
    assertEquals(NativeIO.POSIX.getCacheManipulator().getMemlockLimit() * cluster.getDataNodes().size(),mbs.getAttribute(mxbeanName,"CacheCapacity"));
  }
  finally {
    if (cluster != null) {
      for (      URI dir : cluster.getNameDirs(0)) {
        FileUtil.chmod(new File(new File(dir),"current").getAbsolutePath(),"755");
      }
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestNameNodeRespectsBindHostKeys

APIUtilityVerifier InternalCallVerifier ConditionMatcher 

@Test(timeout=300000) public void testServiceRpcBindHostKey() throws IOException {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  LOG.info("Testing without " + DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY);
  conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,LOCALHOST_SERVER_ADDRESS);
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    String address=getServiceRpcServerAddress(cluster);
    assertThat("Bind address not expected to be wildcard by default.",address,not("/" + WILDCARD_ADDRESS));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
      cluster=null;
    }
  }
  LOG.info("Testing with " + DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY);
  conf.set(DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY,WILDCARD_ADDRESS);
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    String address=getServiceRpcServerAddress(cluster);
    assertThat("Bind address " + address + " is not wildcard.",address,is("/" + WILDCARD_ADDRESS));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier ConditionMatcher 

@Test(timeout=300000) public void testRpcBindHostKey() throws IOException {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  LOG.info("Testing without " + DFS_NAMENODE_RPC_BIND_HOST_KEY);
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    String address=getRpcServerAddress(cluster);
    assertThat("Bind address not expected to be wildcard by default.",address,not("/" + WILDCARD_ADDRESS));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
      cluster=null;
    }
  }
  LOG.info("Testing with " + DFS_NAMENODE_RPC_BIND_HOST_KEY);
  conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY,WILDCARD_ADDRESS);
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    String address=getRpcServerAddress(cluster);
    assertThat("Bind address " + address + " is not wildcard.",address,is("/" + WILDCARD_ADDRESS));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestNameNodeXAttr

APIUtilityVerifier EqualityVerifier 

@Test(timeout=120000) public void testXAttrSymlinks() throws Exception {
  fs.mkdirs(linkParent);
  fs.mkdirs(targetParent);
  DFSTestUtil.createFile(fs,target,1024,(short)3,0xBEEFl);
  fs.createSymlink(target,link,false);
  fs.setXAttr(target,name1,value1);
  fs.setXAttr(target,name2,value2);
  Map xattrs=fs.getXAttrs(link);
  Assert.assertEquals(xattrs.size(),2);
  Assert.assertArrayEquals(value1,xattrs.get(name1));
  Assert.assertArrayEquals(value2,xattrs.get(name2));
  fs.setXAttr(link,name3,null);
  xattrs=fs.getXAttrs(target);
  Assert.assertEquals(xattrs.size(),3);
  Assert.assertArrayEquals(value1,xattrs.get(name1));
  Assert.assertArrayEquals(value2,xattrs.get(name2));
  Assert.assertArrayEquals(new byte[0],xattrs.get(name3));
  fs.removeXAttr(link,name1);
  xattrs=fs.getXAttrs(target);
  Assert.assertEquals(xattrs.size(),2);
  Assert.assertArrayEquals(value2,xattrs.get(name2));
  Assert.assertArrayEquals(new byte[0],xattrs.get(name3));
  fs.removeXAttr(target,name3);
  xattrs=fs.getXAttrs(link);
  Assert.assertEquals(xattrs.size(),1);
  Assert.assertArrayEquals(value2,xattrs.get(name2));
  fs.delete(linkParent,true);
  fs.delete(targetParent,true);
}

Class: org.apache.hadoop.hdfs.server.namenode.TestNamenodeCapacityReport

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier 

@Test public void testXceiverCount() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setInt(DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY,0);
  MiniDFSCluster cluster=null;
  final int nodes=8;
  final int fileCount=5;
  final short fileRepl=3;
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(nodes).build();
    cluster.waitActive();
    final FSNamesystem namesystem=cluster.getNamesystem();
    final DatanodeManager dnm=namesystem.getBlockManager().getDatanodeManager();
    List datanodes=cluster.getDataNodes();
    final DistributedFileSystem fs=cluster.getFileSystem();
    triggerHeartbeats(datanodes);
    int expectedTotalLoad=nodes;
    int expectedInServiceNodes=nodes;
    int expectedInServiceLoad=nodes;
    assertEquals(nodes,namesystem.getNumLiveDataNodes());
    assertEquals(expectedInServiceNodes,namesystem.getNumDatanodesInService());
    assertEquals(expectedTotalLoad,namesystem.getTotalLoad());
    assertEquals((double)expectedInServiceLoad / expectedInServiceLoad,namesystem.getInServiceXceiverAverage(),EPSILON);
    for (int i=0; i < nodes / 2; i++) {
      DataNode dn=datanodes.get(i);
      DatanodeDescriptor dnd=dnm.getDatanode(dn.getDatanodeId());
      dn.shutdown();
      dnd.setLastUpdate(0L);
      BlockManagerTestUtil.checkHeartbeat(namesystem.getBlockManager());
      expectedInServiceNodes--;
      assertEquals(expectedInServiceNodes,namesystem.getNumLiveDataNodes());
      assertEquals(expectedInServiceNodes,namesystem.getNumDatanodesInService());
    }
    cluster.restartDataNodes();
    cluster.waitActive();
    datanodes=cluster.getDataNodes();
    expectedInServiceNodes=nodes;
    assertEquals(nodes,datanodes.size());
    assertEquals(nodes,namesystem.getNumLiveDataNodes());
    assertEquals(expectedInServiceNodes,namesystem.getNumDatanodesInService());
    assertEquals(expectedTotalLoad,namesystem.getTotalLoad());
    assertEquals((double)expectedInServiceLoad / expectedInServiceLoad,namesystem.getInServiceXceiverAverage(),EPSILON);
    DFSOutputStream[] streams=new DFSOutputStream[fileCount];
    for (int i=0; i < fileCount; i++) {
      streams[i]=(DFSOutputStream)fs.create(new Path("/f" + i),fileRepl).getWrappedStream();
      streams[i].write("1".getBytes());
      streams[i].hsync();
      expectedTotalLoad+=2 * fileRepl;
      expectedInServiceLoad+=2 * fileRepl;
    }
    triggerHeartbeats(datanodes);
    assertEquals(nodes,namesystem.getNumLiveDataNodes());
    assertEquals(expectedInServiceNodes,namesystem.getNumDatanodesInService());
    assertEquals(expectedTotalLoad,namesystem.getTotalLoad());
    assertEquals((double)expectedInServiceLoad / expectedInServiceNodes,namesystem.getInServiceXceiverAverage(),EPSILON);
    for (int i=0; i < fileRepl; i++) {
      expectedInServiceNodes--;
      DatanodeDescriptor dnd=dnm.getDatanode(datanodes.get(i).getDatanodeId());
      expectedInServiceLoad-=dnd.getXceiverCount();
      dnm.startDecommission(dnd);
      DataNodeTestUtils.triggerHeartbeat(datanodes.get(i));
      Thread.sleep(100);
      assertEquals(nodes,namesystem.getNumLiveDataNodes());
      assertEquals(expectedInServiceNodes,namesystem.getNumDatanodesInService());
      assertEquals(expectedTotalLoad,namesystem.getTotalLoad());
      assertEquals((double)expectedInServiceLoad / expectedInServiceNodes,namesystem.getInServiceXceiverAverage(),EPSILON);
    }
    for (int i=0; i < fileCount; i++) {
      int decomm=0;
      for (      DatanodeInfo dni : streams[i].getPipeline()) {
        DatanodeDescriptor dnd=dnm.getDatanode(dni);
        expectedTotalLoad-=2;
        if (dnd.isDecommissionInProgress() || dnd.isDecommissioned()) {
          decomm++;
        }
 else {
          expectedInServiceLoad-=2;
        }
      }
      try {
        streams[i].close();
      }
 catch (      IOException ioe) {
        if (decomm < fileRepl) {
          throw ioe;
        }
      }
      triggerHeartbeats(datanodes);
      assertEquals(nodes,namesystem.getNumLiveDataNodes());
      assertEquals(expectedInServiceNodes,namesystem.getNumDatanodesInService());
      assertEquals(expectedTotalLoad,namesystem.getTotalLoad());
      assertEquals((double)expectedInServiceLoad / expectedInServiceNodes,namesystem.getInServiceXceiverAverage(),EPSILON);
    }
    for (int i=0; i < nodes; i++) {
      DataNode dn=datanodes.get(i);
      dn.shutdown();
      DatanodeDescriptor dnDesc=dnm.getDatanode(dn.getDatanodeId());
      dnDesc.setLastUpdate(0L);
      BlockManagerTestUtil.checkHeartbeat(namesystem.getBlockManager());
      assertEquals(nodes - 1 - i,namesystem.getNumLiveDataNodes());
      if (i >= fileRepl) {
        expectedInServiceNodes--;
      }
      assertEquals(expectedInServiceNodes,namesystem.getNumDatanodesInService());
      double expectedXceiverAvg=(i == nodes - 1) ? 0.0 : 1.0;
      assertEquals((double)expectedXceiverAvg,namesystem.getInServiceXceiverAverage(),EPSILON);
    }
    assertEquals(0,namesystem.getNumLiveDataNodes());
    assertEquals(0,namesystem.getNumDatanodesInService());
    assertEquals(0.0,namesystem.getTotalLoad(),EPSILON);
    assertEquals(0.0,namesystem.getInServiceXceiverAverage(),EPSILON);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestNamenodeRetryCache

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * After run a set of operations, restart NN and check if the retry cache has
 * been rebuilt based on the editlog.
 */
@Test public void testRetryCacheRebuild() throws Exception {
  DFSTestUtil.runOperations(cluster,filesystem,conf,BlockSize,0);
  LightWeightCache cacheSet=(LightWeightCache)namesystem.getRetryCache().getCacheSet();
  assertEquals(23,cacheSet.size());
  Map oldEntries=new HashMap();
  Iterator iter=cacheSet.iterator();
  while (iter.hasNext()) {
    CacheEntry entry=iter.next();
    oldEntries.put(entry,entry);
  }
  cluster.restartNameNode();
  cluster.waitActive();
  namesystem=cluster.getNamesystem();
  assertTrue(namesystem.hasRetryCache());
  cacheSet=(LightWeightCache)namesystem.getRetryCache().getCacheSet();
  assertEquals(23,cacheSet.size());
  iter=cacheSet.iterator();
  while (iter.hasNext()) {
    CacheEntry entry=iter.next();
    assertTrue(oldEntries.containsKey(entry));
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestParallelImageWrite

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * check if DFS remains in proper condition after a restart 
 */
@Test public void testRestartDFS() throws Exception {
  final Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  FSNamesystem fsn=null;
  int numNamenodeDirs;
  DFSTestUtil files=new DFSTestUtil.Builder().setName("TestRestartDFS").setNumFiles(200).build();
  final String dir="/srcdat";
  final Path rootpath=new Path("/");
  final Path dirpath=new Path(dir);
  long rootmtime;
  FileStatus rootstatus;
  FileStatus dirstatus;
  try {
    cluster=new MiniDFSCluster.Builder(conf).format(true).numDataNodes(NUM_DATANODES).build();
    String[] nameNodeDirs=conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,new String[]{});
    numNamenodeDirs=nameNodeDirs.length;
    assertTrue("failed to get number of Namenode StorageDirs",numNamenodeDirs != 0);
    FileSystem fs=cluster.getFileSystem();
    files.createFiles(fs,dir);
    rootmtime=fs.getFileStatus(rootpath).getModificationTime();
    rootstatus=fs.getFileStatus(dirpath);
    dirstatus=fs.getFileStatus(dirpath);
    fs.setOwner(rootpath,rootstatus.getOwner() + "_XXX",null);
    fs.setOwner(dirpath,null,dirstatus.getGroup() + "_XXX");
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
  try {
    conf.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY,1);
    cluster=new MiniDFSCluster.Builder(conf).format(false).numDataNodes(NUM_DATANODES).build();
    fsn=cluster.getNamesystem();
    FileSystem fs=cluster.getFileSystem();
    assertTrue("Filesystem corrupted after restart.",files.checkFiles(fs,dir));
    final FileStatus newrootstatus=fs.getFileStatus(rootpath);
    assertEquals(rootmtime,newrootstatus.getModificationTime());
    assertEquals(rootstatus.getOwner() + "_XXX",newrootstatus.getOwner());
    assertEquals(rootstatus.getGroup(),newrootstatus.getGroup());
    final FileStatus newdirstatus=fs.getFileStatus(dirpath);
    assertEquals(dirstatus.getOwner(),newdirstatus.getOwner());
    assertEquals(dirstatus.getGroup() + "_XXX",newdirstatus.getGroup());
    rootmtime=fs.getFileStatus(rootpath).getModificationTime();
    final String checkAfterRestart=checkImages(fsn,numNamenodeDirs);
    files.cleanup(fs,dir);
    files.createFiles(fs,dir);
    fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    cluster.getNameNodeRpc().saveNamespace();
    final String checkAfterModify=checkImages(fsn,numNamenodeDirs);
    assertFalse("Modified namespace should change fsimage contents. " + "was: " + checkAfterRestart + " now: "+ checkAfterModify,checkAfterRestart.equals(checkAfterModify));
    fsn.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    files.cleanup(fs,dir);
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestProcessCorruptBlocks

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * The corrupt block has to be removed when the number of valid replicas
 * matches replication factor for the file. In this the above condition is
 * tested by reducing the replication factor 
 * The test strategy : 
 * Bring up Cluster with 3 DataNodes
 * Create a file of replication factor 3 
 * Corrupt one replica of a block of the file 
 * Verify that there are still 2 good replicas and 1 corrupt replica
 * (corrupt replica should not be removed since number of good
 * replicas (2) is less than replication factor (3))
 * Set the replication factor to 2 
 * Verify that the corrupt replica is removed. 
 * (corrupt replica  should not be removed since number of good
 * replicas (2) is equal to replication factor (2))
 */
@Test public void testWhenDecreasingReplication() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,1000L);
  conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY,Integer.toString(2));
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
  FileSystem fs=cluster.getFileSystem();
  final FSNamesystem namesystem=cluster.getNamesystem();
  try {
    final Path fileName=new Path("/foo1");
    DFSTestUtil.createFile(fs,fileName,2,(short)3,0L);
    DFSTestUtil.waitReplication(fs,fileName,(short)3);
    ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,fileName);
    corruptBlock(cluster,fs,fileName,0,block);
    DFSTestUtil.waitReplication(fs,fileName,(short)2);
    assertEquals(2,countReplicas(namesystem,block).liveReplicas());
    assertEquals(1,countReplicas(namesystem,block).corruptReplicas());
    namesystem.setReplication(fileName.toString(),(short)2);
    try {
      Thread.sleep(3000);
    }
 catch (    InterruptedException ignored) {
    }
    assertEquals(2,countReplicas(namesystem,block).liveReplicas());
    assertEquals(0,countReplicas(namesystem,block).corruptReplicas());
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * None of the blocks can be removed if all blocks are corrupt.
 * The test strategy : 
 * Bring up Cluster with 3 DataNodes
 * Create a file of replication factor 3 
 * Corrupt all three replicas 
 * Verify that all replicas are corrupt and 3 replicas are present.
 * Set the replication factor to 1 
 * Verify that all replicas are corrupt and 3 replicas are present.
 */
@Test public void testWithAllCorruptReplicas() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,1000L);
  conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY,Integer.toString(2));
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
  FileSystem fs=cluster.getFileSystem();
  final FSNamesystem namesystem=cluster.getNamesystem();
  try {
    final Path fileName=new Path("/foo1");
    DFSTestUtil.createFile(fs,fileName,2,(short)3,0L);
    DFSTestUtil.waitReplication(fs,fileName,(short)3);
    ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,fileName);
    corruptBlock(cluster,fs,fileName,0,block);
    corruptBlock(cluster,fs,fileName,1,block);
    corruptBlock(cluster,fs,fileName,2,block);
    try {
      Thread.sleep(3000);
    }
 catch (    InterruptedException ignored) {
    }
    assertEquals(0,countReplicas(namesystem,block).liveReplicas());
    assertEquals(3,countReplicas(namesystem,block).corruptReplicas());
    namesystem.setReplication(fileName.toString(),(short)1);
    try {
      Thread.sleep(3000);
    }
 catch (    InterruptedException ignored) {
    }
    assertEquals(0,countReplicas(namesystem,block).liveReplicas());
    assertEquals(3,countReplicas(namesystem,block).corruptReplicas());
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * The corrupt block has to be removed when the number of valid replicas
 * matches replication factor for the file. In this test, the above 
 * condition is achieved by increasing the number of good replicas by 
 * replicating on a new Datanode. 
 * The test strategy : 
 * Bring up Cluster with 3 DataNodes
 * Create a file  of replication factor 3
 * Corrupt one replica of a block of the file 
 * Verify that there are still 2 good replicas and 1 corrupt replica 
 * (corrupt replica should not be removed since number of good replicas
 * (2) is less  than replication factor (3)) 
 * Start a new data node 
 * Verify that the a new replica is created and corrupt replica is
 * removed.
 */
@Test public void testByAddingAnExtraDataNode() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,1000L);
  conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY,Integer.toString(2));
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
  FileSystem fs=cluster.getFileSystem();
  final FSNamesystem namesystem=cluster.getNamesystem();
  DataNodeProperties dnPropsFourth=cluster.stopDataNode(3);
  try {
    final Path fileName=new Path("/foo1");
    DFSTestUtil.createFile(fs,fileName,2,(short)3,0L);
    DFSTestUtil.waitReplication(fs,fileName,(short)3);
    ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,fileName);
    corruptBlock(cluster,fs,fileName,0,block);
    DFSTestUtil.waitReplication(fs,fileName,(short)2);
    assertEquals(2,countReplicas(namesystem,block).liveReplicas());
    assertEquals(1,countReplicas(namesystem,block).corruptReplicas());
    cluster.restartDataNode(dnPropsFourth);
    DFSTestUtil.waitReplication(fs,fileName,(short)3);
    assertEquals(3,countReplicas(namesystem,block).liveReplicas());
    assertEquals(0,countReplicas(namesystem,block).corruptReplicas());
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * The corrupt block has to be removed when the number of valid replicas
 * matches replication factor for the file. The above condition should hold
 * true as long as there is one good replica. This test verifies that.
 * The test strategy : 
 * Bring up Cluster with 2 DataNodes
 * Create a file of replication factor 2 
 * Corrupt one replica of a block of the file 
 * Verify that there is  one good replicas and 1 corrupt replica 
 * (corrupt replica should not be removed since number of good 
 * replicas (1) is less than replication factor (2)).
 * Set the replication factor to 1 
 * Verify that the corrupt replica is removed. 
 * (corrupt replica should  be removed since number of good
 * replicas (1) is equal to replication factor (1))
 */
@Test(timeout=20000) public void testWithReplicationFactorAsOne() throws Exception {
  Configuration conf=new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY,1000L);
  conf.set(DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY,Integer.toString(2));
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  FileSystem fs=cluster.getFileSystem();
  final FSNamesystem namesystem=cluster.getNamesystem();
  try {
    final Path fileName=new Path("/foo1");
    DFSTestUtil.createFile(fs,fileName,2,(short)2,0L);
    DFSTestUtil.waitReplication(fs,fileName,(short)2);
    ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,fileName);
    corruptBlock(cluster,fs,fileName,0,block);
    DFSTestUtil.waitReplication(fs,fileName,(short)1);
    assertEquals(1,countReplicas(namesystem,block).liveReplicas());
    assertEquals(1,countReplicas(namesystem,block).corruptReplicas());
    namesystem.setReplication(fileName.toString(),(short)1);
    for (int i=0; i < 10; i++) {
      try {
        Thread.sleep(1000);
      }
 catch (      InterruptedException ignored) {
      }
      if (countReplicas(namesystem,block).corruptReplicas() == 0) {
        break;
      }
    }
    assertEquals(1,countReplicas(namesystem,block).liveReplicas());
    assertEquals(0,countReplicas(namesystem,block).corruptReplicas());
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestSaveNamespace

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Verify that a saveNamespace command brings faulty directories
 * in fs.name.dir and fs.edit.dir back online.
 */
@Test(timeout=30000) public void testReinsertnamedirsInSavenamespace() throws Exception {
  Configuration conf=getConf();
  conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY,true);
  NameNode.initMetrics(conf,NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  FSNamesystem fsn=FSNamesystem.loadFromDisk(conf);
  FSImage originalImage=fsn.getFSImage();
  NNStorage storage=originalImage.getStorage();
  FSImage spyImage=spy(originalImage);
  Whitebox.setInternalState(fsn,"fsImage",spyImage);
  FileSystem fs=FileSystem.getLocal(conf);
  File rootDir=storage.getStorageDir(0).getRoot();
  Path rootPath=new Path(rootDir.getPath(),"current");
  final FsPermission permissionNone=new FsPermission((short)0);
  final FsPermission permissionAll=new FsPermission(FsAction.ALL,FsAction.READ_EXECUTE,FsAction.READ_EXECUTE);
  fs.setPermission(rootPath,permissionNone);
  try {
    doAnEdit(fsn,1);
    fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    LOG.info("Doing the first savenamespace.");
    fsn.saveNamespace();
    LOG.info("First savenamespace sucessful.");
    assertTrue("Savenamespace should have marked one directory as bad." + " But found " + storage.getRemovedStorageDirs().size() + " bad directories.",storage.getRemovedStorageDirs().size() == 1);
    fs.setPermission(rootPath,permissionAll);
    LOG.info("Doing the second savenamespace.");
    fsn.saveNamespace();
    LOG.warn("Second savenamespace sucessful.");
    assertTrue("Savenamespace should have been successful in removing " + " bad directories from Image." + " But found " + storage.getRemovedStorageDirs().size() + " bad directories.",storage.getRemovedStorageDirs().size() == 0);
    LOG.info("Shutting down fsimage.");
    originalImage.close();
    fsn.close();
    fsn=null;
    LOG.info("Loading new FSmage from disk.");
    fsn=FSNamesystem.loadFromDisk(conf);
    LOG.info("Checking reloaded image.");
    checkEditExists(fsn,1);
    LOG.info("Reloaded image is good.");
  }
  finally {
    if (rootDir.exists()) {
      fs.setPermission(rootPath,permissionAll);
    }
    if (fsn != null) {
      try {
        fsn.close();
      }
 catch (      Throwable t) {
        LOG.fatal("Failed to shut down",t);
      }
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=30000) public void testTxIdPersistence() throws Exception {
  Configuration conf=getConf();
  NameNode.initMetrics(conf,NamenodeRole.NAMENODE);
  DFSTestUtil.formatNameNode(conf);
  FSNamesystem fsn=FSNamesystem.loadFromDisk(conf);
  try {
    assertEquals(1,fsn.getEditLog().getLastWrittenTxId());
    doAnEdit(fsn,1);
    assertEquals(2,fsn.getEditLog().getLastWrittenTxId());
    fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    fsn.saveNamespace();
    assertEquals(4,fsn.getEditLog().getLastWrittenTxId());
    fsn.getFSImage().close();
    fsn.close();
    assertEquals(5,fsn.getEditLog().getLastWrittenTxId());
    fsn=null;
    fsn=FSNamesystem.loadFromDisk(conf);
    assertEquals(6,fsn.getEditLog().getLastWrittenTxId());
  }
  finally {
    if (fsn != null) {
      fsn.close();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestSecondaryWebUi

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testSecondaryWebUi() throws IOException, MalformedObjectNameException, AttributeNotFoundException, MBeanException, ReflectionException, InstanceNotFoundException {
  MBeanServer mbs=ManagementFactory.getPlatformMBeanServer();
  ObjectName mxbeanName=new ObjectName("Hadoop:service=SecondaryNameNode,name=SecondaryNameNodeInfo");
  String[] checkpointDir=(String[])mbs.getAttribute(mxbeanName,"CheckpointDirectories");
  Assert.assertArrayEquals(checkpointDir,snn.getCheckpointDirectories());
  String[] checkpointEditlogDir=(String[])mbs.getAttribute(mxbeanName,"CheckpointEditlogDirectories");
  Assert.assertArrayEquals(checkpointEditlogDir,snn.getCheckpointEditlogDirectories());
}

Class: org.apache.hadoop.hdfs.server.namenode.TestSecureNameNode

APIUtilityVerifier UtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testName() throws IOException, InterruptedException {
  MiniDFSCluster cluster=null;
  try {
    String keyTabDir=System.getProperty("kdc.resource.dir") + "/keytabs";
    String nn1KeytabPath=keyTabDir + "/nn1.keytab";
    String user1KeyTabPath=keyTabDir + "/user1.keytab";
    Configuration conf=new HdfsConfiguration();
    conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,"kerberos");
    conf.set(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY,"nn1/localhost@EXAMPLE.COM");
    conf.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,nn1KeytabPath);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_OF_DATANODES).build();
    final MiniDFSCluster clusterRef=cluster;
    cluster.waitActive();
    FileSystem fsForCurrentUser=cluster.getFileSystem();
    fsForCurrentUser.mkdirs(new Path("/tmp"));
    fsForCurrentUser.setPermission(new Path("/tmp"),new FsPermission((short)511));
    UserGroupInformation ugi=UserGroupInformation.loginUserFromKeytabAndReturnUGI("user1@EXAMPLE.COM",user1KeyTabPath);
    FileSystem fs=ugi.doAs(new PrivilegedExceptionAction(){
      @Override public FileSystem run() throws Exception {
        return clusterRef.getFileSystem();
      }
    }
);
    try {
      Path p=new Path("/users");
      fs.mkdirs(p);
      fail("user1 must not be allowed to write in /");
    }
 catch (    IOException expected) {
    }
    Path p=new Path("/tmp/alpha");
    fs.mkdirs(p);
    assertNotNull(fs.listStatus(p));
    assertEquals(AuthenticationMethod.KERBEROS,ugi.getAuthenticationMethod());
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestSecureNameNodeWithExternalKdc

APIUtilityVerifier UtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testSecureNameNode() throws IOException, InterruptedException {
  MiniDFSCluster cluster=null;
  try {
    String nnPrincipal=System.getProperty("dfs.namenode.kerberos.principal");
    String nnSpnegoPrincipal=System.getProperty("dfs.namenode.kerberos.internal.spnego.principal");
    String nnKeyTab=System.getProperty("dfs.namenode.keytab.file");
    assertNotNull("NameNode principal was not specified",nnPrincipal);
    assertNotNull("NameNode SPNEGO principal was not specified",nnSpnegoPrincipal);
    assertNotNull("NameNode keytab was not specified",nnKeyTab);
    Configuration conf=new HdfsConfiguration();
    conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,"kerberos");
    conf.set(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY,nnPrincipal);
    conf.set(DFSConfigKeys.DFS_NAMENODE_KERBEROS_INTERNAL_SPNEGO_PRINCIPAL_KEY,nnSpnegoPrincipal);
    conf.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,nnKeyTab);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_OF_DATANODES).build();
    final MiniDFSCluster clusterRef=cluster;
    cluster.waitActive();
    FileSystem fsForCurrentUser=cluster.getFileSystem();
    fsForCurrentUser.mkdirs(new Path("/tmp"));
    fsForCurrentUser.setPermission(new Path("/tmp"),new FsPermission((short)511));
    String userPrincipal=System.getProperty("user.principal");
    String userKeyTab=System.getProperty("user.keytab");
    assertNotNull("User principal was not specified",userPrincipal);
    assertNotNull("User keytab was not specified",userKeyTab);
    UserGroupInformation ugi=UserGroupInformation.loginUserFromKeytabAndReturnUGI(userPrincipal,userKeyTab);
    FileSystem fs=ugi.doAs(new PrivilegedExceptionAction(){
      @Override public FileSystem run() throws Exception {
        return clusterRef.getFileSystem();
      }
    }
);
    try {
      Path p=new Path("/users");
      fs.mkdirs(p);
      fail("User must not be allowed to write in /");
    }
 catch (    IOException expected) {
    }
    Path p=new Path("/tmp/alpha");
    fs.mkdirs(p);
    assertNotNull(fs.listStatus(p));
    assertEquals(AuthenticationMethod.KERBEROS,ugi.getAuthenticationMethod());
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestSecurityTokenEditLog

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Tests transaction logging in dfs.
 */
@Test public void testEditLog() throws IOException {
  Configuration conf=new HdfsConfiguration();
  MiniDFSCluster cluster=null;
  FileSystem fileSys=null;
  try {
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,true);
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
    cluster.waitActive();
    fileSys=cluster.getFileSystem();
    final FSNamesystem namesystem=cluster.getNamesystem();
    for (Iterator it=cluster.getNameDirs(0).iterator(); it.hasNext(); ) {
      File dir=new File(it.next().getPath());
      System.out.println(dir);
    }
    FSImage fsimage=namesystem.getFSImage();
    FSEditLog editLog=fsimage.getEditLog();
    editLog.setOutputBufferCapacity(2048);
    Thread threadId[]=new Thread[NUM_THREADS];
    for (int i=0; i < NUM_THREADS; i++) {
      Transactions trans=new Transactions(namesystem,NUM_TRANSACTIONS);
      threadId[i]=new Thread(trans,"TransactionThread-" + i);
      threadId[i].start();
    }
    for (int i=0; i < NUM_THREADS; i++) {
      try {
        threadId[i].join();
      }
 catch (      InterruptedException e) {
        i--;
      }
    }
    editLog.close();
    namesystem.getDelegationTokenSecretManager().stopThreads();
    int numKeys=namesystem.getDelegationTokenSecretManager().getNumberOfKeys();
    int expectedTransactions=NUM_THREADS * opsPerTrans * NUM_TRANSACTIONS + numKeys + 2;
    for (    StorageDirectory sd : fsimage.getStorage().dirIterable(NameNodeDirType.EDITS)) {
      File editFile=NNStorage.getFinalizedEditsFile(sd,1,1 + expectedTransactions - 1);
      System.out.println("Verifying file: " + editFile);
      FSEditLogLoader loader=new FSEditLogLoader(namesystem,0);
      long numEdits=loader.loadFSEdits(new EditLogFileInputStream(editFile),1);
      assertEquals("Verification for " + editFile,expectedTransactions,numEdits);
    }
  }
  finally {
    if (fileSys != null)     fileSys.close();
    if (cluster != null)     cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestSequentialBlockId

APIUtilityVerifier InternalCallVerifier ConditionMatcher 

/** 
 * Test that collisions in the block ID space are handled gracefully.
 * @throws IOException
 */
@Test public void testTriggerBlockIdCollision() throws IOException {
  Configuration conf=new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY,1);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    FSNamesystem fsn=cluster.getNamesystem();
    final int blockCount=10;
    Path path1=new Path("testBlockIdCollisionDetection_file1.dat");
    DFSTestUtil.createFile(fs,path1,IO_SIZE,BLOCK_SIZE * blockCount,BLOCK_SIZE,REPLICATION,SEED);
    List blocks1=DFSTestUtil.getAllBlocks(fs,path1);
    SequentialBlockIdGenerator blockIdGenerator=fsn.getBlockIdGenerator();
    blockIdGenerator.setCurrentValue(blockIdGenerator.getCurrentValue() - 5);
    Path path2=new Path("testBlockIdCollisionDetection_file2.dat");
    DFSTestUtil.createFile(fs,path2,IO_SIZE,BLOCK_SIZE * blockCount,BLOCK_SIZE,REPLICATION,SEED);
    List blocks2=DFSTestUtil.getAllBlocks(fs,path2);
    assertThat(blocks2.size(),is(blockCount));
    assertThat(blocks2.get(0).getBlock().getBlockId(),is(blocks1.get(9).getBlock().getBlockId() + 1));
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier ConditionMatcher 

/** 
 * Test that block IDs are generated sequentially.
 * @throws IOException
 */
@Test public void testBlockIdGeneration() throws IOException {
  Configuration conf=new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY,1);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    cluster.waitActive();
    FileSystem fs=cluster.getFileSystem();
    Path path=new Path("testBlockIdGeneration.dat");
    DFSTestUtil.createFile(fs,path,IO_SIZE,BLOCK_SIZE * 10,BLOCK_SIZE,REPLICATION,SEED);
    List blocks=DFSTestUtil.getAllBlocks(fs,path);
    LOG.info("Block0 id is " + blocks.get(0).getBlock().getBlockId());
    long nextBlockExpectedId=blocks.get(0).getBlock().getBlockId() + 1;
    for (int i=1; i < blocks.size(); ++i) {
      long nextBlockId=blocks.get(i).getBlock().getBlockId();
      LOG.info("Block" + i + " id is "+ nextBlockId);
      assertThat(nextBlockId,is(nextBlockExpectedId));
      ++nextBlockExpectedId;
    }
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.TestSnapshotPathINodes

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Test {@link INodeDirectory#getExistingPathINodes(byte[][],int,boolean)} 
 * for snapshot file while modifying file after snapshot.
 */
@Test(timeout=15000) public void testSnapshotPathINodesAfterModification() throws Exception {
  String[] names=INode.getPathNames(file1.toString());
  byte[][] components=INode.getPathComponents(names);
  INodesInPath nodesInPath=INodesInPath.resolve(fsdir.rootDir,components);
  INode[] inodes=nodesInPath.getINodes();
  assertEquals(inodes.length,components.length);
  assertEquals(inodes[components.length - 1].getFullPathName(),file1.toString());
  final long modTime=inodes[inodes.length - 1].getModificationTime();
  hdfs.allowSnapshot(sub1);
  hdfs.createSnapshot(sub1,"s3");
  DFSTestUtil.appendFile(hdfs,file1,"the content for appending");
  String snapshotPath=sub1.toString() + "/.snapshot/s3/file1";
  names=INode.getPathNames(snapshotPath);
  components=INode.getPathComponents(names);
  INodesInPath ssNodesInPath=INodesInPath.resolve(fsdir.rootDir,components);
  INode[] ssInodes=ssNodesInPath.getINodes();
  assertEquals(ssInodes.length,components.length - 1);
  final Snapshot s3=getSnapshot(ssNodesInPath,"s3");
  assertSnapshot(ssNodesInPath,true,s3,3);
  INode snapshotFileNode=ssInodes[ssInodes.length - 1];
  assertEquals(snapshotFileNode.getLocalName(),file1.getName());
  assertTrue(snapshotFileNode.asFile().isWithSnapshot());
  assertEquals(modTime,snapshotFileNode.getModificationTime(ssNodesInPath.getPathSnapshotId()));
  names=INode.getPathNames(file1.toString());
  components=INode.getPathComponents(names);
  INodesInPath newNodesInPath=INodesInPath.resolve(fsdir.rootDir,components);
  assertSnapshot(newNodesInPath,false,s3,-1);
  INode[] newInodes=newNodesInPath.getINodes();
  assertEquals(newInodes.length,components.length);
  final int last=components.length - 1;
  assertEquals(newInodes[last].getFullPathName(),file1.toString());
  Assert.assertFalse(modTime == newInodes[last].getModificationTime());
  hdfs.deleteSnapshot(sub1,"s3");
  hdfs.disallowSnapshot(sub1);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Test {@link INodeDirectory#getExistingPathINodes(byte[][],int,boolean)} 
 * for normal (non-snapshot) file.
 */
@Test(timeout=15000) public void testNonSnapshotPathINodes() throws Exception {
  String[] names=INode.getPathNames(file1.toString());
  byte[][] components=INode.getPathComponents(names);
  INodesInPath nodesInPath=INodesInPath.resolve(fsdir.rootDir,components);
  INode[] inodes=nodesInPath.getINodes();
  assertEquals(inodes.length,components.length);
  assertSnapshot(nodesInPath,false,null,-1);
  assertTrue("file1=" + file1 + ", nodesInPath="+ nodesInPath,inodes[components.length - 1] != null);
  assertEquals(inodes[components.length - 1].getFullPathName(),file1.toString());
  assertEquals(inodes[components.length - 2].getFullPathName(),sub1.toString());
  assertEquals(inodes[components.length - 3].getFullPathName(),dir.toString());
  nodesInPath=INodesInPath.resolve(fsdir.rootDir,components,1,false);
  inodes=nodesInPath.getINodes();
  assertEquals(inodes.length,1);
  assertSnapshot(nodesInPath,false,null,-1);
  assertEquals(inodes[0].getFullPathName(),file1.toString());
  nodesInPath=INodesInPath.resolve(fsdir.rootDir,components,2,false);
  inodes=nodesInPath.getINodes();
  assertEquals(inodes.length,2);
  assertSnapshot(nodesInPath,false,null,-1);
  assertEquals(inodes[1].getFullPathName(),file1.toString());
  assertEquals(inodes[0].getFullPathName(),sub1.toString());
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Test {@link INodeDirectory#getExistingPathINodes(byte[][],int,boolean)} 
 * for snapshot file while adding a new file after snapshot.
 */
@Test(timeout=15000) public void testSnapshotPathINodesWithAddedFile() throws Exception {
  hdfs.allowSnapshot(sub1);
  hdfs.createSnapshot(sub1,"s4");
  final Path file3=new Path(sub1,"file3");
  DFSTestUtil.createFile(hdfs,file3,1024,REPLICATION,seed);
{
    String snapshotPath=sub1.toString() + "/.snapshot/s4/file3";
    String[] names=INode.getPathNames(snapshotPath);
    byte[][] components=INode.getPathComponents(names);
    INodesInPath nodesInPath=INodesInPath.resolve(fsdir.rootDir,components);
    INode[] inodes=nodesInPath.getINodes();
    assertEquals(inodes.length,components.length - 1);
    assertEquals(nodesInPath.getNumNonNull(),components.length - 2);
    s4=getSnapshot(nodesInPath,"s4");
    assertSnapshot(nodesInPath,true,s4,3);
    assertNull(inodes[inodes.length - 1]);
  }
  String[] names=INode.getPathNames(file3.toString());
  byte[][] components=INode.getPathComponents(names);
  INodesInPath nodesInPath=INodesInPath.resolve(fsdir.rootDir,components);
  INode[] inodes=nodesInPath.getINodes();
  assertEquals(inodes.length,components.length);
  assertSnapshot(nodesInPath,false,s4,-1);
  assertEquals(inodes[components.length - 1].getFullPathName(),file3.toString());
  assertEquals(inodes[components.length - 2].getFullPathName(),sub1.toString());
  assertEquals(inodes[components.length - 3].getFullPathName(),dir.toString());
  hdfs.deleteSnapshot(sub1,"s4");
  hdfs.disallowSnapshot(sub1);
}

APIUtilityVerifier IterativeVerifier UtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Test {@link INodeDirectory#getExistingPathINodes(byte[][],int,boolean)} 
 * for snapshot file.
 */
@Test(timeout=15000) public void testSnapshotPathINodes() throws Exception {
  hdfs.allowSnapshot(sub1);
  hdfs.createSnapshot(sub1,"s1");
  String snapshotPath=sub1.toString() + "/.snapshot/s1/file1";
  String[] names=INode.getPathNames(snapshotPath);
  byte[][] components=INode.getPathComponents(names);
  INodesInPath nodesInPath=INodesInPath.resolve(fsdir.rootDir,components);
  INode[] inodes=nodesInPath.getINodes();
  assertEquals(inodes.length,components.length - 1);
  final Snapshot snapshot=getSnapshot(nodesInPath,"s1");
  assertSnapshot(nodesInPath,true,snapshot,3);
  INode snapshotFileNode=inodes[inodes.length - 1];
  assertINodeFile(snapshotFileNode,file1);
  assertTrue(snapshotFileNode.getParent().isWithSnapshot());
  nodesInPath=INodesInPath.resolve(fsdir.rootDir,components,1,false);
  inodes=nodesInPath.getINodes();
  assertEquals(inodes.length,1);
  assertSnapshot(nodesInPath,true,snapshot,-1);
  assertINodeFile(nodesInPath.getLastINode(),file1);
  nodesInPath=INodesInPath.resolve(fsdir.rootDir,components,2,false);
  inodes=nodesInPath.getINodes();
  assertEquals(inodes.length,2);
  assertSnapshot(nodesInPath,true,snapshot,0);
  assertINodeFile(nodesInPath.getLastINode(),file1);
  String dotSnapshotPath=sub1.toString() + "/.snapshot";
  names=INode.getPathNames(dotSnapshotPath);
  components=INode.getPathComponents(names);
  nodesInPath=INodesInPath.resolve(fsdir.rootDir,components);
  inodes=nodesInPath.getINodes();
  assertEquals(inodes.length,components.length - 1);
  assertSnapshot(nodesInPath,true,snapshot,-1);
  final INode last=nodesInPath.getLastINode();
  assertEquals(last.getFullPathName(),sub1.toString());
  assertFalse(last instanceof INodeFile);
  String[] invalidPathComponent={"invalidDir","foo",".snapshot","bar"};
  Path invalidPath=new Path(invalidPathComponent[0]);
  for (int i=1; i < invalidPathComponent.length; i++) {
    invalidPath=new Path(invalidPath,invalidPathComponent[i]);
    try {
      hdfs.getFileStatus(invalidPath);
      Assert.fail();
    }
 catch (    FileNotFoundException fnfe) {
      System.out.println("The exception is expected: " + fnfe);
    }
  }
  hdfs.deleteSnapshot(sub1,"s1");
  hdfs.disallowSnapshot(sub1);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier PublicFieldVerifier HybridVerifier 

/** 
 * Test {@link INodeDirectory#getExistingPathINodes(byte[][],int,boolean)} 
 * for snapshot file after deleting the original file.
 */
@Test(timeout=15000) public void testSnapshotPathINodesAfterDeletion() throws Exception {
  hdfs.allowSnapshot(sub1);
  hdfs.createSnapshot(sub1,"s2");
  hdfs.delete(file1,false);
  final Snapshot snapshot;
{
    String snapshotPath=sub1.toString() + "/.snapshot/s2/file1";
    String[] names=INode.getPathNames(snapshotPath);
    byte[][] components=INode.getPathComponents(names);
    INodesInPath nodesInPath=INodesInPath.resolve(fsdir.rootDir,components);
    INode[] inodes=nodesInPath.getINodes();
    assertEquals(inodes.length,components.length - 1);
    snapshot=getSnapshot(nodesInPath,"s2");
    assertSnapshot(nodesInPath,true,snapshot,3);
    final INode inode=inodes[inodes.length - 1];
    assertEquals(file1.getName(),inode.getLocalName());
    assertTrue(inode.asFile().isWithSnapshot());
  }
  String[] names=INode.getPathNames(file1.toString());
  byte[][] components=INode.getPathComponents(names);
  INodesInPath nodesInPath=INodesInPath.resolve(fsdir.rootDir,components);
  INode[] inodes=nodesInPath.getINodes();
  assertEquals(inodes.length,components.length);
  assertEquals(nodesInPath.getNumNonNull(),components.length - 1);
  assertSnapshot(nodesInPath,false,snapshot,-1);
  assertNull(inodes[components.length - 1]);
  assertEquals(inodes[components.length - 2].getFullPathName(),sub1.toString());
  assertEquals(inodes[components.length - 3].getFullPathName(),dir.toString());
  hdfs.deleteSnapshot(sub1,"s2");
  hdfs.disallowSnapshot(sub1);
}

Class: org.apache.hadoop.hdfs.server.namenode.TestStartup

APIUtilityVerifier EqualityVerifier 

/** 
 * Verify the following scenario.
 * 1. NN restarts.
 * 2. Heartbeat RPC will retry and succeed. NN asks DN to reregister.
 * 3. After reregistration completes, DN will send Heartbeat, followed by
 * Blockreport.
 * 4. NN will mark DatanodeStorageInfo#blockContentsStale to false.
 * @throws Exception
 */
@Test(timeout=60000) public void testStorageBlockContentsStaleAfterNNRestart() throws Exception {
  MiniDFSCluster dfsCluster=null;
  try {
    Configuration config=new Configuration();
    dfsCluster=new MiniDFSCluster.Builder(config).numDataNodes(1).build();
    dfsCluster.waitActive();
    dfsCluster.restartNameNode(true);
    BlockManagerTestUtil.checkHeartbeat(dfsCluster.getNamesystem().getBlockManager());
    MBeanServer mbs=ManagementFactory.getPlatformMBeanServer();
    ObjectName mxbeanNameFsns=new ObjectName("Hadoop:service=NameNode,name=FSNamesystemState");
    Integer numStaleStorages=(Integer)(mbs.getAttribute(mxbeanNameFsns,"NumStaleStorages"));
    assertEquals(0,numStaleStorages.intValue());
  }
  finally {
    if (dfsCluster != null) {
      dfsCluster.shutdown();
    }
  }
  return;
}

Class: org.apache.hadoop.hdfs.server.namenode.TestStorageRestore

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * 1. create DFS cluster with 3 storage directories
 * - 2 EDITS_IMAGE(name1, name2), 1 EDITS(name3)
 * 2. create a file
 * 3. corrupt/disable name2 and name3 by removing rwx permission
 * 4. run doCheckpoint
 * - will fail on removed dirs (which invalidates them)
 * 5. write another file
 * 6. check there is only one healthy storage dir
 * 7. run doCheckpoint - recover should fail but checkpoint should succeed
 * 8. check there is still only one healthy storage dir
 * 9. restore the access permission for name2 and name 3, run checkpoint again
 * 10.verify there are 3 healthy storage dirs.
 */
@Test public void testStorageRestoreFailure() throws Exception {
  SecondaryNameNode secondary=null;
  String nameDir2=Shell.WINDOWS ? (new File(path2,"current").getAbsolutePath()) : path2.toString();
  String nameDir3=Shell.WINDOWS ? (new File(path3,"current").getAbsolutePath()) : path3.toString();
  try {
    cluster=new MiniDFSCluster.Builder(config).numDataNodes(0).manageNameDfsDirs(false).build();
    cluster.waitActive();
    secondary=new SecondaryNameNode(config);
    printStorages(cluster.getNameNode().getFSImage());
    FileSystem fs=cluster.getFileSystem();
    Path path=new Path("/","test");
    assertTrue(fs.mkdirs(path));
    assertTrue(FileUtil.chmod(nameDir2,"000") == 0);
    assertTrue(FileUtil.chmod(nameDir3,"000") == 0);
    secondary.doCheckpoint();
    printStorages(cluster.getNameNode().getFSImage());
    path=new Path("/","test1");
    assertTrue(fs.mkdirs(path));
    assert (cluster.getNameNode().getFSImage().getStorage().getNumStorageDirs() == 1);
    secondary.doCheckpoint();
    assert (cluster.getNameNode().getFSImage().getStorage().getNumStorageDirs() == 1);
    assertTrue(FileUtil.chmod(nameDir2,"755") == 0);
    assertTrue(FileUtil.chmod(nameDir3,"755") == 0);
    secondary.doCheckpoint();
    assert (cluster.getNameNode().getFSImage().getStorage().getNumStorageDirs() == 3);
  }
  finally {
    if (path2.exists()) {
      FileUtil.chmod(nameDir2,"755");
    }
    if (path3.exists()) {
      FileUtil.chmod(nameDir3,"755");
    }
    if (cluster != null) {
      cluster.shutdown();
    }
    if (secondary != null) {
      secondary.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestBootstrapStandby

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test for downloading a checkpoint made at a later checkpoint
 * from the active.
 */
@Test public void testDownloadingLaterCheckpoint() throws Exception {
  nn0.getRpcServer().rollEditLog();
  nn0.getRpcServer().rollEditLog();
  NameNodeAdapter.enterSafeMode(nn0,false);
  NameNodeAdapter.saveNamespace(nn0);
  NameNodeAdapter.leaveSafeMode(nn0);
  long expectedCheckpointTxId=NameNodeAdapter.getNamesystem(nn0).getFSImage().getMostRecentCheckpointTxId();
  assertEquals(6,expectedCheckpointTxId);
  int rc=BootstrapStandby.run(new String[]{"-force"},cluster.getConfiguration(1));
  assertEquals(0,rc);
  FSImageTestUtil.assertNNHasCheckpoints(cluster,1,ImmutableList.of((int)expectedCheckpointTxId));
  FSImageTestUtil.assertNNFilesMatch(cluster);
  cluster.restartNameNode(1);
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestBootstrapStandbyWithQJM

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * BootstrapStandby when the existing NN is standby 
 */
@Test public void testBootstrapStandbyWithStandbyNN() throws Exception {
  cluster.transitionToStandby(0);
  Configuration confNN1=cluster.getConfiguration(1);
  cluster.shutdownNameNode(1);
  int rc=BootstrapStandby.run(new String[]{"-force"},confNN1);
  assertEquals(0,rc);
  FSImageTestUtil.assertNNHasCheckpoints(cluster,1,ImmutableList.of(0));
  FSImageTestUtil.assertNNFilesMatch(cluster);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * BootstrapStandby when the existing NN is active 
 */
@Test public void testBootstrapStandbyWithActiveNN() throws Exception {
  cluster.transitionToActive(0);
  Configuration confNN1=cluster.getConfiguration(1);
  cluster.shutdownNameNode(1);
  int rc=BootstrapStandby.run(new String[]{"-force"},confNN1);
  assertEquals(0,rc);
  FSImageTestUtil.assertNNHasCheckpoints(cluster,1,ImmutableList.of(0));
  FSImageTestUtil.assertNNFilesMatch(cluster);
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestDFSUpgradeWithHA

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * Make sure that starting a second NN with the -upgrade flag fails if the
 * other NN has already done that.
 */
@Test public void testCannotUpgradeSecondNameNode() throws IOException, URISyntaxException {
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0).build();
    File sharedDir=new File(cluster.getSharedEditsDir(0,1));
    checkClusterPreviousDirExistence(cluster,false);
    assertCTimesEqual(cluster);
    checkPreviousDirExistence(sharedDir,false);
    cluster.transitionToActive(0);
    fs=HATestUtil.configureFailoverFs(cluster,conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0,false);
    checkNnPreviousDirExistence(cluster,0,true);
    checkNnPreviousDirExistence(cluster,1,false);
    checkPreviousDirExistence(sharedDir,true);
    assertTrue(fs.mkdirs(new Path("/foo2")));
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
    cluster.restartNameNode(0,false);
    cluster.transitionToActive(0);
    assertTrue(fs.mkdirs(new Path("/foo3")));
    cluster.getNameNodeInfos()[1].setStartOpt(StartupOption.UPGRADE);
    try {
      cluster.restartNameNode(1,false);
      fail("Should not have been able to start second NN with -upgrade");
    }
 catch (    IOException ioe) {
      GenericTestUtils.assertExceptionContains("It looks like the shared log is already being upgraded",ioe);
    }
  }
  finally {
    if (fs != null) {
      fs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Ensure that an admin cannot finalize an HA upgrade without at least one NN
 * being active.
 */
@Test public void testCannotFinalizeIfNoActive() throws IOException, URISyntaxException {
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0).build();
    File sharedDir=new File(cluster.getSharedEditsDir(0,1));
    checkClusterPreviousDirExistence(cluster,false);
    assertCTimesEqual(cluster);
    checkPreviousDirExistence(sharedDir,false);
    cluster.transitionToActive(0);
    fs=HATestUtil.configureFailoverFs(cluster,conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0,false);
    checkNnPreviousDirExistence(cluster,0,true);
    checkNnPreviousDirExistence(cluster,1,false);
    checkPreviousDirExistence(sharedDir,true);
    assertTrue(fs.mkdirs(new Path("/foo2")));
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
    cluster.restartNameNode(0,false);
    cluster.transitionToActive(0);
    assertTrue(fs.mkdirs(new Path("/foo3")));
    int rc=BootstrapStandby.run(new String[]{"-force"},cluster.getConfiguration(1));
    assertEquals(0,rc);
    cluster.restartNameNode(1);
    cluster.transitionToStandby(0);
    cluster.transitionToActive(1);
    assertTrue(fs.mkdirs(new Path("/foo4")));
    assertCTimesEqual(cluster);
    cluster.transitionToStandby(1);
    try {
      runFinalizeCommand(cluster);
      fail("Should not have been able to finalize upgrade with no NN active");
    }
 catch (    IOException ioe) {
      GenericTestUtils.assertExceptionContains("Cannot finalize with no NameNode active",ioe);
    }
  }
  finally {
    if (fs != null) {
      fs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Make sure that an HA NN with NFS-based HA can successfully start and
 * upgrade.
 */
@Test public void testNfsUpgrade() throws IOException, URISyntaxException {
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0).build();
    File sharedDir=new File(cluster.getSharedEditsDir(0,1));
    checkClusterPreviousDirExistence(cluster,false);
    assertCTimesEqual(cluster);
    checkPreviousDirExistence(sharedDir,false);
    cluster.transitionToActive(0);
    fs=HATestUtil.configureFailoverFs(cluster,conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0,false);
    checkNnPreviousDirExistence(cluster,0,true);
    checkNnPreviousDirExistence(cluster,1,false);
    checkPreviousDirExistence(sharedDir,true);
    assertTrue(fs.mkdirs(new Path("/foo2")));
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
    cluster.restartNameNode(0,false);
    cluster.transitionToActive(0);
    assertTrue(fs.mkdirs(new Path("/foo3")));
    int rc=BootstrapStandby.run(new String[]{"-force"},cluster.getConfiguration(1));
    assertEquals(0,rc);
    cluster.restartNameNode(1);
    cluster.transitionToStandby(0);
    cluster.transitionToActive(1);
    assertTrue(fs.mkdirs(new Path("/foo4")));
    assertCTimesEqual(cluster);
  }
  finally {
    if (fs != null) {
      fs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testFinalizeWithJournalNodes() throws IOException, URISyntaxException {
  MiniQJMHACluster qjCluster=null;
  FileSystem fs=null;
  try {
    Builder builder=new MiniQJMHACluster.Builder(conf);
    builder.getDfsBuilder().numDataNodes(0);
    qjCluster=builder.build();
    MiniDFSCluster cluster=qjCluster.getDfsCluster();
    checkJnPreviousDirExistence(qjCluster,false);
    checkClusterPreviousDirExistence(cluster,false);
    assertCTimesEqual(cluster);
    cluster.transitionToActive(0);
    fs=HATestUtil.configureFailoverFs(cluster,conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0,false);
    assertTrue(fs.mkdirs(new Path("/foo2")));
    checkNnPreviousDirExistence(cluster,0,true);
    checkNnPreviousDirExistence(cluster,1,false);
    checkJnPreviousDirExistence(qjCluster,true);
    int rc=BootstrapStandby.run(new String[]{"-force"},cluster.getConfiguration(1));
    assertEquals(0,rc);
    cluster.restartNameNode(1);
    runFinalizeCommand(cluster);
    checkClusterPreviousDirExistence(cluster,false);
    checkJnPreviousDirExistence(qjCluster,false);
    assertCTimesEqual(cluster);
  }
  finally {
    if (fs != null) {
      fs.close();
    }
    if (qjCluster != null) {
      qjCluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Make sure that an HA NN can successfully upgrade when configured using
 * JournalNodes.
 */
@Test public void testUpgradeWithJournalNodes() throws IOException, URISyntaxException {
  MiniQJMHACluster qjCluster=null;
  FileSystem fs=null;
  try {
    Builder builder=new MiniQJMHACluster.Builder(conf);
    builder.getDfsBuilder().numDataNodes(0);
    qjCluster=builder.build();
    MiniDFSCluster cluster=qjCluster.getDfsCluster();
    checkJnPreviousDirExistence(qjCluster,false);
    checkClusterPreviousDirExistence(cluster,false);
    assertCTimesEqual(cluster);
    cluster.transitionToActive(0);
    fs=HATestUtil.configureFailoverFs(cluster,conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0,false);
    checkNnPreviousDirExistence(cluster,0,true);
    checkNnPreviousDirExistence(cluster,1,false);
    checkJnPreviousDirExistence(qjCluster,true);
    assertTrue(fs.mkdirs(new Path("/foo2")));
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.REGULAR);
    cluster.restartNameNode(0,false);
    cluster.transitionToActive(0);
    assertTrue(fs.mkdirs(new Path("/foo3")));
    int rc=BootstrapStandby.run(new String[]{"-force"},cluster.getConfiguration(1));
    assertEquals(0,rc);
    cluster.restartNameNode(1);
    cluster.transitionToStandby(0);
    cluster.transitionToActive(1);
    assertTrue(fs.mkdirs(new Path("/foo4")));
    assertCTimesEqual(cluster);
  }
  finally {
    if (fs != null) {
      fs.close();
    }
    if (qjCluster != null) {
      qjCluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test rollback with NFS shared dir.
 */
@Test public void testRollbackWithNfs() throws Exception {
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0).build();
    File sharedDir=new File(cluster.getSharedEditsDir(0,1));
    checkClusterPreviousDirExistence(cluster,false);
    assertCTimesEqual(cluster);
    checkPreviousDirExistence(sharedDir,false);
    cluster.transitionToActive(0);
    fs=HATestUtil.configureFailoverFs(cluster,conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0,false);
    checkNnPreviousDirExistence(cluster,0,true);
    checkNnPreviousDirExistence(cluster,1,false);
    checkPreviousDirExistence(sharedDir,true);
    assertTrue(fs.mkdirs(new Path("/foo2")));
    int rc=BootstrapStandby.run(new String[]{"-force"},cluster.getConfiguration(1));
    assertEquals(0,rc);
    cluster.restartNameNode(1);
    checkNnPreviousDirExistence(cluster,0,true);
    checkNnPreviousDirExistence(cluster,1,false);
    checkPreviousDirExistence(sharedDir,true);
    assertCTimesEqual(cluster);
    Collection nn1NameDirs=cluster.getNameDirs(0);
    cluster.shutdown();
    conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,Joiner.on(",").join(nn1NameDirs));
    NameNode.doRollback(conf,false);
    checkNnPreviousDirExistence(cluster,0,false);
    checkPreviousDirExistence(sharedDir,false);
  }
  finally {
    if (fs != null) {
      fs.close();
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testRollbackWithJournalNodes() throws IOException, URISyntaxException {
  MiniQJMHACluster qjCluster=null;
  FileSystem fs=null;
  try {
    Builder builder=new MiniQJMHACluster.Builder(conf);
    builder.getDfsBuilder().numDataNodes(0);
    qjCluster=builder.build();
    MiniDFSCluster cluster=qjCluster.getDfsCluster();
    checkClusterPreviousDirExistence(cluster,false);
    assertCTimesEqual(cluster);
    checkJnPreviousDirExistence(qjCluster,false);
    cluster.transitionToActive(0);
    fs=HATestUtil.configureFailoverFs(cluster,conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0,false);
    checkNnPreviousDirExistence(cluster,0,true);
    checkNnPreviousDirExistence(cluster,1,false);
    checkJnPreviousDirExistence(qjCluster,true);
    assertTrue(fs.mkdirs(new Path("/foo2")));
    int rc=BootstrapStandby.run(new String[]{"-force"},cluster.getConfiguration(1));
    assertEquals(0,rc);
    cluster.restartNameNode(1);
    checkNnPreviousDirExistence(cluster,0,true);
    checkNnPreviousDirExistence(cluster,1,false);
    checkJnPreviousDirExistence(qjCluster,true);
    assertCTimesEqual(cluster);
    Collection nn1NameDirs=cluster.getNameDirs(0);
    cluster.shutdown();
    conf.setStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY,Joiner.on(",").join(nn1NameDirs));
    NameNode.doRollback(conf,false);
    checkNnPreviousDirExistence(cluster,0,false);
    checkJnPreviousDirExistence(qjCluster,false);
  }
  finally {
    if (fs != null) {
      fs.close();
    }
    if (qjCluster != null) {
      qjCluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Make sure that even if the NN which initiated the upgrade is in the standby
 * state that we're allowed to finalize.
 */
@Test public void testFinalizeFromSecondNameNodeWithJournalNodes() throws IOException, URISyntaxException {
  MiniQJMHACluster qjCluster=null;
  FileSystem fs=null;
  try {
    Builder builder=new MiniQJMHACluster.Builder(conf);
    builder.getDfsBuilder().numDataNodes(0);
    qjCluster=builder.build();
    MiniDFSCluster cluster=qjCluster.getDfsCluster();
    checkJnPreviousDirExistence(qjCluster,false);
    checkClusterPreviousDirExistence(cluster,false);
    assertCTimesEqual(cluster);
    cluster.transitionToActive(0);
    fs=HATestUtil.configureFailoverFs(cluster,conf);
    assertTrue(fs.mkdirs(new Path("/foo1")));
    cluster.shutdownNameNode(1);
    cluster.getNameNodeInfos()[0].setStartOpt(StartupOption.UPGRADE);
    cluster.restartNameNode(0,false);
    checkNnPreviousDirExistence(cluster,0,true);
    checkNnPreviousDirExistence(cluster,1,false);
    checkJnPreviousDirExistence(qjCluster,true);
    int rc=BootstrapStandby.run(new String[]{"-force"},cluster.getConfiguration(1));
    assertEquals(0,rc);
    cluster.restartNameNode(1);
    cluster.transitionToStandby(0);
    cluster.transitionToActive(1);
    runFinalizeCommand(cluster);
    checkClusterPreviousDirExistence(cluster,false);
    checkJnPreviousDirExistence(qjCluster,false);
    assertCTimesEqual(cluster);
  }
  finally {
    if (fs != null) {
      fs.close();
    }
    if (qjCluster != null) {
      qjCluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestDelegationTokensWithHA

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * Test if StandbyException can be thrown from StandbyNN, when it's requested for 
 * password. (HDFS-6475). With StandbyException, the client can failover to try
 * activeNN.
 */
@Test public void testDelegationTokenStandbyNNAppearFirst() throws Exception {
  cluster.transitionToStandby(0);
  cluster.transitionToActive(1);
  final DelegationTokenSecretManager stSecretManager=NameNodeAdapter.getDtSecretManager(nn1.getNamesystem());
  final Token token=getDelegationToken(fs,"JobTracker");
  final DelegationTokenIdentifier identifier=new DelegationTokenIdentifier();
  byte[] tokenId=token.getIdentifier();
  identifier.readFields(new DataInputStream(new ByteArrayInputStream(tokenId)));
  assertTrue(null != stSecretManager.retrievePassword(identifier));
  final UserGroupInformation ugi=UserGroupInformation.createRemoteUser("JobTracker");
  ugi.addToken(token);
  ugi.doAs(new PrivilegedExceptionAction(){
    @Override public Object run(){
      try {
        try {
          byte[] tmppw=dtSecretManager.retrievePassword(identifier);
          fail("InvalidToken with cause StandbyException is expected" + " since nn0 is standby");
          return tmppw;
        }
 catch (        IOException e) {
          throw new SecurityException("Failed to obtain user group information: " + e,e);
        }
      }
 catch (      Exception oe) {
        HttpServletResponse response=mock(HttpServletResponse.class);
        ExceptionHandler eh=new ExceptionHandler();
        eh.initResponse(response);
        Response resp=eh.toResponse(oe);
        Map m=(Map)JSON.parse(resp.getEntity().toString());
        RemoteException re=JsonUtil.toRemoteException(m);
        Exception unwrapped=((RemoteException)re).unwrapRemoteException(StandbyException.class);
        assertTrue(unwrapped instanceof StandbyException);
        return null;
      }
    }
  }
);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * HDFS-3062: DistributedFileSystem.getCanonicalServiceName() throws an
 * exception if the URI is a logical URI. This bug fails the combination of
 * ha + mapred + security.
 */
@Test public void testDFSGetCanonicalServiceName() throws Exception {
  URI hAUri=HATestUtil.getLogicalUri(cluster);
  String haService=HAUtil.buildTokenServiceForLogicalUri(hAUri,HdfsConstants.HDFS_URI_SCHEME).toString();
  assertEquals(haService,dfs.getCanonicalServiceName());
  final String renewer=UserGroupInformation.getCurrentUser().getShortUserName();
  final Token token=getDelegationToken(dfs,renewer);
  assertEquals(haService,token.getService().toString());
  token.renew(dfs.getConf());
  token.cancel(dfs.getConf());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testHdfsGetCanonicalServiceName() throws Exception {
  Configuration conf=dfs.getConf();
  URI haUri=HATestUtil.getLogicalUri(cluster);
  AbstractFileSystem afs=AbstractFileSystem.createFileSystem(haUri,conf);
  String haService=HAUtil.buildTokenServiceForLogicalUri(haUri,HdfsConstants.HDFS_URI_SCHEME).toString();
  assertEquals(haService,afs.getCanonicalServiceName());
  Token token=afs.getDelegationTokens(UserGroupInformation.getCurrentUser().getShortUserName()).get(0);
  assertEquals(haService,token.getService().toString());
  token.renew(conf);
  token.cancel(conf);
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testHAUtilClonesDelegationTokens() throws Exception {
  final Token token=getDelegationToken(fs,"JobTracker");
  UserGroupInformation ugi=UserGroupInformation.createRemoteUser("test");
  URI haUri=new URI("hdfs://my-ha-uri/");
  token.setService(HAUtil.buildTokenServiceForLogicalUri(haUri,HdfsConstants.HDFS_URI_SCHEME));
  ugi.addToken(token);
  Collection nnAddrs=new HashSet();
  nnAddrs.add(new InetSocketAddress("localhost",nn0.getNameNodeAddress().getPort()));
  nnAddrs.add(new InetSocketAddress("localhost",nn1.getNameNodeAddress().getPort()));
  HAUtil.cloneDelegationTokenForLogicalUri(ugi,haUri,nnAddrs);
  Collection> tokens=ugi.getTokens();
  assertEquals(3,tokens.size());
  LOG.info("Tokens:\n" + Joiner.on("\n").join(tokens));
  DelegationTokenSelector dts=new DelegationTokenSelector();
  for (  InetSocketAddress addr : nnAddrs) {
    Text ipcDtService=SecurityUtil.buildTokenService(addr);
    Token token2=dts.selectToken(ipcDtService,ugi.getTokens());
    assertNotNull(token2);
    assertArrayEquals(token.getIdentifier(),token2.getIdentifier());
    assertArrayEquals(token.getPassword(),token2.getPassword());
  }
  SecurityUtilTestHelper.setTokenServiceUseIp(false);
  for (  InetSocketAddress addr : nnAddrs) {
    Text ipcDtService=SecurityUtil.buildTokenService(addr);
    Token token2=dts.selectToken(ipcDtService,ugi.getTokens());
    assertNull(token2);
  }
  HAUtil.cloneDelegationTokenForLogicalUri(ugi,haUri,nnAddrs);
  for (  InetSocketAddress addr : nnAddrs) {
    Text ipcDtService=SecurityUtil.buildTokenService(addr);
    Token token2=dts.selectToken(ipcDtService,ugi.getTokens());
    assertNotNull(token2);
    assertArrayEquals(token.getIdentifier(),token2.getIdentifier());
    assertArrayEquals(token.getPassword(),token2.getPassword());
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestFailoverWithBlockTokensEnabled

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

@Test public void ensureInvalidBlockTokensAreRejected() throws IOException, URISyntaxException {
  cluster.transitionToActive(0);
  FileSystem fs=HATestUtil.configureFailoverFs(cluster,conf);
  DFSTestUtil.writeFile(fs,TEST_PATH,TEST_DATA);
  assertEquals(TEST_DATA,DFSTestUtil.readFile(fs,TEST_PATH));
  DFSClient dfsClient=DFSClientAdapter.getDFSClient((DistributedFileSystem)fs);
  DFSClient spyDfsClient=Mockito.spy(dfsClient);
  Mockito.doAnswer(new Answer(){
    @Override public LocatedBlocks answer(    InvocationOnMock arg0) throws Throwable {
      LocatedBlocks locatedBlocks=(LocatedBlocks)arg0.callRealMethod();
      for (      LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
        Token token=lb.getBlockToken();
        BlockTokenIdentifier id=lb.getBlockToken().decodeIdentifier();
        id.setExpiryDate(Time.now() + 10);
        Token newToken=new Token(id.getBytes(),token.getPassword(),token.getKind(),token.getService());
        lb.setBlockToken(newToken);
      }
      return locatedBlocks;
    }
  }
).when(spyDfsClient).getLocatedBlocks(Mockito.anyString(),Mockito.anyLong(),Mockito.anyLong());
  DFSClientAdapter.setDFSClient((DistributedFileSystem)fs,spyDfsClient);
  try {
    assertEquals(TEST_DATA,DFSTestUtil.readFile(fs,TEST_PATH));
    fail("Shouldn't have been able to read a file with invalid block tokens");
  }
 catch (  IOException ioe) {
    GenericTestUtils.assertExceptionContains("Could not obtain block",ioe);
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestFailureOfSharedDir

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test that marking the shared edits dir as being "required" causes the NN to
 * fail if that dir can't be accessed.
 */
@Test public void testFailureOfSharedDir() throws Exception {
  Configuration conf=new Configuration();
  conf.setLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,2000);
  MiniDFSCluster cluster=null;
  File sharedEditsDir=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0).checkExitOnShutdown(false).build();
    cluster.waitActive();
    cluster.transitionToActive(0);
    FileSystem fs=HATestUtil.configureFailoverFs(cluster,conf);
    assertTrue(fs.mkdirs(new Path("/test1")));
    URI sharedEditsUri=cluster.getSharedEditsDir(0,1);
    sharedEditsDir=new File(sharedEditsUri);
    assertEquals(0,FileUtil.chmod(sharedEditsDir.getAbsolutePath(),"-w",true));
    Thread.sleep(conf.getLong(DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_DEFAULT) * 2);
    NameNode nn1=cluster.getNameNode(1);
    assertTrue(nn1.isStandbyState());
    assertFalse("StandBy NameNode should not go to SafeMode on resource unavailability",nn1.isInSafeMode());
    NameNode nn0=cluster.getNameNode(0);
    try {
      nn0.getRpcServer().rollEditLog();
      fail("Succeeded in rolling edit log despite shared dir being deleted");
    }
 catch (    ExitException ee) {
      GenericTestUtils.assertExceptionContains("finalize log segment 1, 3 failed for required journal",ee);
    }
    for (    URI editsUri : cluster.getNameEditsDirs(0)) {
      if (editsUri.equals(sharedEditsUri)) {
        continue;
      }
      File editsDir=new File(editsUri.getPath());
      File curDir=new File(editsDir,"current");
      GenericTestUtils.assertGlobEquals(curDir,"edits_.*",NNStorage.getInProgressEditsFileName(1));
    }
  }
  finally {
    if (sharedEditsDir != null) {
      FileUtil.chmod(sharedEditsDir.getAbsolutePath(),"+w",true);
    }
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Test that the shared edits dir is automatically added to the list of edits
 * dirs that are marked required.
 */
@Test public void testSharedDirIsAutomaticallyMarkedRequired() throws URISyntaxException {
  URI foo=new URI("file:/foo");
  URI bar=new URI("file:/bar");
  Configuration conf=new Configuration();
  conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,Joiner.on(",").join(foo,bar));
  conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_REQUIRED_KEY,foo.toString());
  assertFalse(FSNamesystem.getRequiredNamespaceEditsDirs(conf).contains(bar));
  conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,bar.toString());
  Collection requiredEditsDirs=FSNamesystem.getRequiredNamespaceEditsDirs(conf);
  assertTrue(Joiner.on(",").join(requiredEditsDirs) + " does not contain " + bar,requiredEditsDirs.contains(bar));
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Make sure that the shared edits dirs are listed before non-shared dirs
 * when the configuration is parsed. This ensures that the shared journals
 * are synced before the local ones.
 */
@Test public void testSharedDirsComeFirstInEditsList() throws Exception {
  Configuration conf=new Configuration();
  URI sharedA=new URI("file:///shared-A");
  URI localA=new URI("file:///local-A");
  URI localB=new URI("file:///local-B");
  URI localC=new URI("file:///local-C");
  conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY,sharedA.toString());
  conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,Joiner.on(",").join(localC,localB,localA));
  List dirs=FSNamesystem.getNamespaceEditsDirs(conf);
  assertEquals("Shared dirs should come first, then local dirs, in the order " + "they were listed in the configuration.",Joiner.on(",").join(sharedA,localC,localB,localA),Joiner.on(",").join(dirs));
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestFailureToReadEdits

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * Test the following case:
 * 1. SBN is reading a finalized edits file when NFS disappears halfway
 * through (or some intermittent error happens)
 * 2. SBN performs a checkpoint and uploads it to the NN
 * 3. NN receives a checkpoint that doesn't correspond to the end of any log
 * segment
 * 4. Both NN and SBN should be able to restart at this point.
 * This is a regression test for HDFS-2766.
 */
@Test public void testCheckpointStartingMidEditsFile() throws Exception {
  assertTrue(fs.mkdirs(new Path(TEST_DIR1)));
  HATestUtil.waitForStandbyToCatchUp(nn0,nn1);
  HATestUtil.waitForCheckpoint(cluster,1,ImmutableList.of(0,3));
  HATestUtil.waitForCheckpoint(cluster,0,ImmutableList.of(0,3));
  causeFailureOnEditLogRead();
  assertTrue(fs.mkdirs(new Path(TEST_DIR2)));
  assertTrue(fs.mkdirs(new Path(TEST_DIR3)));
  try {
    HATestUtil.waitForStandbyToCatchUp(nn0,nn1);
    fail("Standby fully caught up, but should not have been able to");
  }
 catch (  HATestUtil.CouldNotCatchUpException e) {
  }
  HATestUtil.waitForCheckpoint(cluster,1,ImmutableList.of(0,3,5));
  HATestUtil.waitForCheckpoint(cluster,0,ImmutableList.of(0,3,5));
  cluster.restartNameNode(0);
  HATestUtil.waitForCheckpoint(cluster,0,ImmutableList.of(0,3,5));
  FileSystem fs0=null;
  try {
    fs0=FileSystem.get(NameNode.getUri(nn0.getNameNodeAddress()),conf);
    assertTrue(fs0.exists(new Path(TEST_DIR1)));
    assertTrue(fs0.exists(new Path(TEST_DIR2)));
    assertTrue(fs0.exists(new Path(TEST_DIR3)));
  }
  finally {
    if (fs0 != null)     fs0.close();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestHAConfiguration

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Tests that the namenode edits dirs and shared edits dirs are gotten with
 * duplicates removed
 */
@Test public void testHAUniqueEditDirs() throws IOException {
  Configuration conf=new Configuration();
  conf.set(DFS_NAMENODE_EDITS_DIR_KEY,"file://edits/dir, " + "file://edits/shared/dir");
  conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY,"file://edits/shared/dir");
  Collection editsDirs=FSNamesystem.getNamespaceEditsDirs(conf);
  assertEquals(2,editsDirs.size());
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestHAStateTransitions

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test for HDFS-2812. Since lease renewals go from the client
 * only to the active NN, the SBN will have out-of-date lease
 * info when it becomes active. We need to make sure we don't
 * accidentally mark the leases as expired when the failover
 * proceeds.
 */
@Test(timeout=120000) public void testLeasesRenewedOnTransition() throws Exception {
  Configuration conf=new Configuration();
  conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,1);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(1).build();
  FSDataOutputStream stm=null;
  FileSystem fs=HATestUtil.configureFailoverFs(cluster,conf);
  NameNode nn0=cluster.getNameNode(0);
  NameNode nn1=cluster.getNameNode(1);
  try {
    cluster.waitActive();
    cluster.transitionToActive(0);
    LOG.info("Starting with NN 0 active");
    stm=fs.create(TEST_FILE_PATH);
    long nn0t0=NameNodeAdapter.getLeaseRenewalTime(nn0,TEST_FILE_STR);
    assertTrue(nn0t0 > 0);
    long nn1t0=NameNodeAdapter.getLeaseRenewalTime(nn1,TEST_FILE_STR);
    assertEquals("Lease should not yet exist on nn1",-1,nn1t0);
    Thread.sleep(5);
    HATestUtil.waitForStandbyToCatchUp(nn0,nn1);
    long nn1t1=NameNodeAdapter.getLeaseRenewalTime(nn1,TEST_FILE_STR);
    assertTrue("Lease should have been created on standby. Time was: " + nn1t1,nn1t1 > nn0t0);
    Thread.sleep(5);
    LOG.info("Failing over to NN 1");
    cluster.transitionToStandby(0);
    cluster.transitionToActive(1);
    long nn1t2=NameNodeAdapter.getLeaseRenewalTime(nn1,TEST_FILE_STR);
    assertTrue("Lease should have been renewed by failover process",nn1t2 > nn1t1);
  }
  finally {
    IOUtils.closeStream(stm);
    cluster.shutdown();
  }
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * This test also serves to test{@link HAUtil#getProxiesForAllNameNodesInNameservice(Configuration,String)} and{@link DFSUtil#getRpcAddressesForNameserviceId(Configuration,String,String)}by virtue of the fact that it wouldn't work properly if the proxies
 * returned were not for the correct NNs.
 */
@Test public void testIsAtLeastOneActive() throws Exception {
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(new HdfsConfiguration()).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(0).build();
  try {
    Configuration conf=new HdfsConfiguration();
    HATestUtil.setFailoverConfigurations(cluster,conf);
    List namenodes=HAUtil.getProxiesForAllNameNodesInNameservice(conf,HATestUtil.getLogicalHostname(cluster));
    assertEquals(2,namenodes.size());
    assertFalse(HAUtil.isAtLeastOneActive(namenodes));
    cluster.transitionToActive(0);
    assertTrue(HAUtil.isAtLeastOneActive(namenodes));
    cluster.transitionToStandby(0);
    assertFalse(HAUtil.isAtLeastOneActive(namenodes));
    cluster.transitionToActive(1);
    assertTrue(HAUtil.isAtLeastOneActive(namenodes));
    cluster.transitionToStandby(1);
    assertFalse(HAUtil.isAtLeastOneActive(namenodes));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestPendingCorruptDnMessages

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testChangedStorageId() throws IOException, URISyntaxException, InterruptedException {
  HdfsConfiguration conf=new HdfsConfiguration();
  conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,1);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).nnTopology(MiniDFSNNTopology.simpleHATopology()).build();
  try {
    cluster.transitionToActive(0);
    FileSystem fs=HATestUtil.configureFailoverFs(cluster,conf);
    OutputStream out=fs.create(filePath);
    out.write("foo bar baz".getBytes());
    out.close();
    HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0),cluster.getNameNode(1));
    ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,filePath);
    assertTrue(MiniDFSCluster.changeGenStampOfBlock(0,block,900));
    DataNodeProperties dnProps=cluster.stopDataNode(0);
    cluster.restartNameNode(1,false);
    assertTrue(cluster.restartDataNode(dnProps,true));
    while (cluster.getNamesystem(1).getBlockManager().getPendingDataNodeMessageCount() < 1) {
      ThreadUtil.sleepAtLeastIgnoreInterrupts(1000);
    }
    assertEquals(1,cluster.getNamesystem(1).getBlockManager().getPendingDataNodeMessageCount());
    String oldStorageId=getRegisteredDatanodeUid(cluster,1);
    assertTrue(wipeAndRestartDn(cluster,0));
    String newStorageId="";
    do {
      ThreadUtil.sleepAtLeastIgnoreInterrupts(1000);
      newStorageId=getRegisteredDatanodeUid(cluster,1);
      System.out.println("====> oldStorageId: " + oldStorageId + " newStorageId: "+ newStorageId);
    }
 while (newStorageId.equals(oldStorageId));
    assertEquals(0,cluster.getNamesystem(1).getBlockManager().getPendingDataNodeMessageCount());
    cluster.transitionToStandby(0);
    cluster.transitionToActive(1);
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestPipelinesFailover

APIUtilityVerifier BranchVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * Test the scenario where the NN fails over after issuing a block
 * synchronization request, but before it is committed. The
 * DN running the recovery should then fail to commit the synchronization
 * and a later retry will succeed.
 */
@Test(timeout=30000) public void testFailoverRightBeforeCommitSynchronization() throws Exception {
  final Configuration conf=new Configuration();
  conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY,false);
  conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,BLOCK_SIZE);
  FSDataOutputStream stm=null;
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(3).build();
  try {
    cluster.waitActive();
    cluster.transitionToActive(0);
    Thread.sleep(500);
    LOG.info("Starting with NN 0 active");
    FileSystem fs=HATestUtil.configureFailoverFs(cluster,conf);
    stm=fs.create(TEST_PATH);
    AppendTestUtil.write(stm,0,BLOCK_SIZE / 2);
    stm.hflush();
    NameNode nn0=cluster.getNameNode(0);
    ExtendedBlock blk=DFSTestUtil.getFirstBlock(fs,TEST_PATH);
    DatanodeDescriptor expectedPrimary=DFSTestUtil.getExpectedPrimaryNode(nn0,blk);
    LOG.info("Expecting block recovery to be triggered on DN " + expectedPrimary);
    DataNode primaryDN=cluster.getDataNode(expectedPrimary.getIpcPort());
    DatanodeProtocolClientSideTranslatorPB nnSpy=DataNodeTestUtils.spyOnBposToNN(primaryDN,nn0);
    DelayAnswer delayer=new DelayAnswer(LOG);
    Mockito.doAnswer(delayer).when(nnSpy).commitBlockSynchronization(Mockito.eq(blk),Mockito.anyInt(),Mockito.anyLong(),Mockito.eq(true),Mockito.eq(false),(DatanodeID[])Mockito.anyObject(),(String[])Mockito.anyObject());
    DistributedFileSystem fsOtherUser=createFsAsOtherUser(cluster,conf);
    assertFalse(fsOtherUser.recoverLease(TEST_PATH));
    LOG.info("Waiting for commitBlockSynchronization call from primary");
    delayer.waitForCall();
    LOG.info("Failing over to NN 1");
    cluster.transitionToStandby(0);
    cluster.transitionToActive(1);
    delayer.proceed();
    delayer.waitForResult();
    Throwable t=delayer.getThrown();
    if (t == null) {
      fail("commitBlockSynchronization call did not fail on standby");
    }
    GenericTestUtils.assertExceptionContains("Operation category WRITE is not supported",t);
    loopRecoverLease(fsOtherUser,TEST_PATH);
    AppendTestUtil.check(fs,TEST_PATH,BLOCK_SIZE / 2);
  }
  finally {
    IOUtils.closeStream(stm);
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Tests lease recovery if a client crashes. This approximates the
 * use case of HBase WALs being recovered after a NN failover.
 */
@Test(timeout=30000) public void testLeaseRecoveryAfterFailover() throws Exception {
  final Configuration conf=new Configuration();
  conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY,false);
  conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,BLOCK_SIZE);
  FSDataOutputStream stm=null;
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(3).build();
  try {
    cluster.waitActive();
    cluster.transitionToActive(0);
    Thread.sleep(500);
    LOG.info("Starting with NN 0 active");
    FileSystem fs=HATestUtil.configureFailoverFs(cluster,conf);
    stm=fs.create(TEST_PATH);
    AppendTestUtil.write(stm,0,BLOCK_AND_A_HALF);
    stm.hflush();
    LOG.info("Failing over to NN 1");
    cluster.transitionToStandby(0);
    cluster.transitionToActive(1);
    assertTrue(fs.exists(TEST_PATH));
    FileSystem fsOtherUser=createFsAsOtherUser(cluster,conf);
    loopRecoverLease(fsOtherUser,TEST_PATH);
    AppendTestUtil.check(fs,TEST_PATH,BLOCK_AND_A_HALF);
    cluster.transitionToStandby(1);
    cluster.transitionToActive(0);
    AppendTestUtil.check(fs,TEST_PATH,BLOCK_AND_A_HALF);
  }
  finally {
    IOUtils.closeStream(stm);
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestQuotasWithHA

APIUtilityVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier 

/** 
 * Test that quotas are properly tracked by the standby through
 * create, append, delete.
 */
@Test(timeout=60000) public void testQuotasTrackedOnStandby() throws Exception {
  fs.mkdirs(TEST_DIR);
  DistributedFileSystem dfs=(DistributedFileSystem)fs;
  dfs.setQuota(TEST_DIR,NS_QUOTA,DS_QUOTA);
  long expectedSize=3 * BLOCK_SIZE + BLOCK_SIZE / 2;
  DFSTestUtil.createFile(fs,TEST_FILE,expectedSize,(short)1,1L);
  HATestUtil.waitForStandbyToCatchUp(nn0,nn1);
  ContentSummary cs=nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
  assertEquals(NS_QUOTA,cs.getQuota());
  assertEquals(DS_QUOTA,cs.getSpaceQuota());
  assertEquals(expectedSize,cs.getSpaceConsumed());
  assertEquals(1,cs.getDirectoryCount());
  assertEquals(1,cs.getFileCount());
  FSDataOutputStream stm=fs.append(TEST_FILE);
  try {
    byte[] data=new byte[(int)(BLOCK_SIZE * 3 / 2)];
    stm.write(data);
    expectedSize+=data.length;
  }
  finally {
    IOUtils.closeStream(stm);
  }
  HATestUtil.waitForStandbyToCatchUp(nn0,nn1);
  cs=nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
  assertEquals(NS_QUOTA,cs.getQuota());
  assertEquals(DS_QUOTA,cs.getSpaceQuota());
  assertEquals(expectedSize,cs.getSpaceConsumed());
  assertEquals(1,cs.getDirectoryCount());
  assertEquals(1,cs.getFileCount());
  fs.delete(TEST_FILE,true);
  expectedSize=0;
  HATestUtil.waitForStandbyToCatchUp(nn0,nn1);
  cs=nn1.getRpcServer().getContentSummary(TEST_DIR_STR);
  assertEquals(NS_QUOTA,cs.getQuota());
  assertEquals(DS_QUOTA,cs.getSpaceQuota());
  assertEquals(expectedSize,cs.getSpaceConsumed());
  assertEquals(1,cs.getDirectoryCount());
  assertEquals(0,cs.getFileCount());
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestRetryCacheWithHA

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * 1. Run a set of operations
 * 2. Trigger the NN failover
 * 3. Check the retry cache on the original standby NN
 */
@Test(timeout=60000) public void testRetryCacheOnStandbyNN() throws Exception {
  DFSTestUtil.runOperations(cluster,dfs,conf,BlockSize,0);
  FSNamesystem fsn0=cluster.getNamesystem(0);
  LightWeightCache cacheSet=(LightWeightCache)fsn0.getRetryCache().getCacheSet();
  assertEquals(23,cacheSet.size());
  Map oldEntries=new HashMap();
  Iterator iter=cacheSet.iterator();
  while (iter.hasNext()) {
    CacheEntry entry=iter.next();
    oldEntries.put(entry,entry);
  }
  cluster.getNameNode(0).getRpcServer().rollEditLog();
  cluster.getNameNode(1).getNamesystem().getEditLogTailer().doTailEdits();
  cluster.shutdownNameNode(0);
  cluster.transitionToActive(1);
  FSNamesystem fsn1=cluster.getNamesystem(1);
  cacheSet=(LightWeightCache)fsn1.getRetryCache().getCacheSet();
  assertEquals(23,cacheSet.size());
  iter=cacheSet.iterator();
  while (iter.hasNext()) {
    CacheEntry entry=iter.next();
    assertTrue(oldEntries.containsKey(entry));
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.ha.TestStandbyIsHot

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Regression test for HDFS-2795:
 * - Start an HA cluster with a DN.
 * - Write several blocks to the FS with replication 1.
 * - Shutdown the DN
 * - Wait for the NNs to declare the DN dead. All blocks will be under-replicated.
 * - Restart the DN.
 * In the bug, the standby node would only very slowly notice the blocks returning
 * to the cluster.
 */
@Test(timeout=60000) public void testDatanodeRestarts() throws Exception {
  Configuration conf=new Configuration();
  conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,1024);
  HAUtil.setAllowStandbyReads(conf,true);
  conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,0);
  conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,1);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(1).build();
  try {
    NameNode nn0=cluster.getNameNode(0);
    NameNode nn1=cluster.getNameNode(1);
    cluster.transitionToActive(0);
    DFSTestUtil.createFile(cluster.getFileSystem(0),TEST_FILE_PATH,5 * 1024,(short)1,1L);
    HATestUtil.waitForStandbyToCatchUp(nn0,nn1);
    DataNode dn=cluster.getDataNodes().get(0);
    String dnName=dn.getDatanodeId().getXferAddr();
    DataNodeProperties dnProps=cluster.stopDataNode(0);
    BlockManagerTestUtil.noticeDeadDatanode(nn0,dnName);
    BlockManagerTestUtil.noticeDeadDatanode(nn1,dnName);
    BlockManagerTestUtil.updateState(nn0.getNamesystem().getBlockManager());
    BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
    assertEquals(5,nn0.getNamesystem().getUnderReplicatedBlocks());
    assertEquals(0,nn1.getNamesystem().getUnderReplicatedBlocks());
    LocatedBlocks locs=nn1.getRpcServer().getBlockLocations(TEST_FILE,0,1);
    assertEquals("Standby should have registered that the block has no replicas",0,locs.get(0).getLocations().length);
    cluster.restartDataNode(dnProps);
    cluster.waitActive(0);
    cluster.waitActive(1);
    BlockManagerTestUtil.updateState(nn0.getNamesystem().getBlockManager());
    BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
    assertEquals(0,nn0.getNamesystem().getUnderReplicatedBlocks());
    assertEquals(0,nn1.getNamesystem().getUnderReplicatedBlocks());
    locs=nn1.getRpcServer().getBlockLocations(TEST_FILE,0,1);
    assertEquals("Standby should have registered that the block has replicas again",1,locs.get(0).getLocations().length);
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.metrics.TestNameNodeMetrics

APIUtilityVerifier BooleanVerifier 

/** 
 * Test NN checkpoint and transaction-related metrics.
 */
@Test public void testTransactionAndCheckpointMetrics() throws Exception {
  long lastCkptTime=MetricsAsserts.getLongGauge("LastCheckpointTime",getMetrics(NS_METRICS));
  assertGauge("LastCheckpointTime",lastCkptTime,getMetrics(NS_METRICS));
  assertGauge("LastWrittenTransactionId",1L,getMetrics(NS_METRICS));
  assertGauge("TransactionsSinceLastCheckpoint",1L,getMetrics(NS_METRICS));
  assertGauge("TransactionsSinceLastLogRoll",1L,getMetrics(NS_METRICS));
  fs.mkdirs(new Path(TEST_ROOT_DIR_PATH,"/tmp"));
  assertGauge("LastCheckpointTime",lastCkptTime,getMetrics(NS_METRICS));
  assertGauge("LastWrittenTransactionId",2L,getMetrics(NS_METRICS));
  assertGauge("TransactionsSinceLastCheckpoint",2L,getMetrics(NS_METRICS));
  assertGauge("TransactionsSinceLastLogRoll",2L,getMetrics(NS_METRICS));
  cluster.getNameNodeRpc().rollEditLog();
  assertGauge("LastCheckpointTime",lastCkptTime,getMetrics(NS_METRICS));
  assertGauge("LastWrittenTransactionId",4L,getMetrics(NS_METRICS));
  assertGauge("TransactionsSinceLastCheckpoint",4L,getMetrics(NS_METRICS));
  assertGauge("TransactionsSinceLastLogRoll",1L,getMetrics(NS_METRICS));
  cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_ENTER,false);
  cluster.getNameNodeRpc().saveNamespace();
  cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_LEAVE,false);
  long newLastCkptTime=MetricsAsserts.getLongGauge("LastCheckpointTime",getMetrics(NS_METRICS));
  assertTrue(lastCkptTime < newLastCkptTime);
  assertGauge("LastWrittenTransactionId",6L,getMetrics(NS_METRICS));
  assertGauge("TransactionsSinceLastCheckpoint",1L,getMetrics(NS_METRICS));
  assertGauge("TransactionsSinceLastLogRoll",1L,getMetrics(NS_METRICS));
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestINodeFileUnderConstructionWithSnapshot

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * call DFSClient#callGetBlockLocations(...) for snapshot file. Make sure only
 * blocks within the size range are returned.
 */
@Test public void testGetBlockLocations() throws Exception {
  final Path root=new Path("/");
  final Path file=new Path("/file");
  DFSTestUtil.createFile(hdfs,file,BLOCKSIZE,REPLICATION,seed);
  SnapshotTestHelper.createSnapshot(hdfs,root,"s1");
  final Path fileInSnapshot=SnapshotTestHelper.getSnapshotPath(root,"s1",file.getName());
  FileStatus status=hdfs.getFileStatus(fileInSnapshot);
  assertEquals(BLOCKSIZE,status.getLen());
  DFSTestUtil.appendFile(hdfs,file,BLOCKSIZE - 1);
  status=hdfs.getFileStatus(fileInSnapshot);
  assertEquals(BLOCKSIZE,status.getLen());
  status=hdfs.getFileStatus(file);
  assertEquals(BLOCKSIZE * 2 - 1,status.getLen());
  LocatedBlocks blocks=DFSClientAdapter.callGetBlockLocations(cluster.getNameNodeRpc(),fileInSnapshot.toString(),0,Long.MAX_VALUE);
  List blockList=blocks.getLocatedBlocks();
  assertEquals(BLOCKSIZE,blocks.getFileLength());
  assertEquals(1,blockList.size());
  LocatedBlock lastBlock=blocks.getLastLocatedBlock();
  assertEquals(0,lastBlock.getStartOffset());
  assertEquals(BLOCKSIZE,lastBlock.getBlockSize());
  SnapshotTestHelper.createSnapshot(hdfs,root,"s2");
  final Path fileInSnapshot2=SnapshotTestHelper.getSnapshotPath(root,"s2",file.getName());
  HdfsDataOutputStream out=appendFileWithoutClosing(file,BLOCKSIZE);
  out.hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
  status=hdfs.getFileStatus(fileInSnapshot2);
  assertEquals(BLOCKSIZE * 2 - 1,status.getLen());
  status=hdfs.getFileStatus(file);
  assertEquals(BLOCKSIZE * 3 - 1,status.getLen());
  blocks=DFSClientAdapter.callGetBlockLocations(cluster.getNameNodeRpc(),fileInSnapshot2.toString(),0,Long.MAX_VALUE);
  assertFalse(blocks.isUnderConstruction());
  assertTrue(blocks.isLastBlockComplete());
  blockList=blocks.getLocatedBlocks();
  assertEquals(BLOCKSIZE * 2 - 1,blocks.getFileLength());
  assertEquals(2,blockList.size());
  lastBlock=blocks.getLastLocatedBlock();
  assertEquals(BLOCKSIZE,lastBlock.getStartOffset());
  assertEquals(BLOCKSIZE,lastBlock.getBlockSize());
  blocks=DFSClientAdapter.callGetBlockLocations(cluster.getNameNodeRpc(),fileInSnapshot2.toString(),BLOCKSIZE,0);
  blockList=blocks.getLocatedBlocks();
  assertEquals(1,blockList.size());
  blocks=DFSClientAdapter.callGetBlockLocations(cluster.getNameNodeRpc(),file.toString(),0,Long.MAX_VALUE);
  blockList=blocks.getLocatedBlocks();
  assertEquals(3,blockList.size());
  assertTrue(blocks.isUnderConstruction());
  assertFalse(blocks.isLastBlockComplete());
  lastBlock=blocks.getLastLocatedBlock();
  assertEquals(BLOCKSIZE * 2,lastBlock.getStartOffset());
  assertEquals(BLOCKSIZE - 1,lastBlock.getBlockSize());
  out.close();
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestNestedSnapshots

APIUtilityVerifier IterativeVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

/** 
 * Test the snapshot limit of a single snapshottable directory.
 * @throws Exception
 */
@Test(timeout=300000) public void testSnapshotLimit() throws Exception {
  final int step=1000;
  final String dirStr="/testSnapshotLimit/dir";
  final Path dir=new Path(dirStr);
  hdfs.mkdirs(dir,new FsPermission((short)0777));
  hdfs.allowSnapshot(dir);
  int s=0;
  for (; s < SNAPSHOT_LIMIT; s++) {
    final String snapshotName="s" + s;
    hdfs.createSnapshot(dir,snapshotName);
    if (s % step == 0) {
      final Path file=new Path(dirStr,"f" + s);
      DFSTestUtil.createFile(hdfs,file,BLOCKSIZE,REPLICATION,SEED);
    }
  }
  try {
    hdfs.createSnapshot(dir,"s" + s);
    Assert.fail("Expected to fail to create snapshot, but didn't.");
  }
 catch (  IOException ioe) {
    SnapshotTestHelper.LOG.info("The exception is expected.",ioe);
  }
  for (int f=0; f < SNAPSHOT_LIMIT; f+=step) {
    final String file="f" + f;
    s=RANDOM.nextInt(step);
    for (; s < SNAPSHOT_LIMIT; s+=RANDOM.nextInt(step)) {
      final Path p=SnapshotTestHelper.getSnapshotPath(dir,"s" + s,file);
      Assert.assertEquals(s > f,hdfs.exists(p));
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test {@link Snapshot#ID_COMPARATOR}.
 */
@Test(timeout=300000) public void testIdCmp(){
  final PermissionStatus perm=PermissionStatus.createImmutable("user","group",FsPermission.createImmutable((short)0));
  final INodeDirectory snapshottable=new INodeDirectory(0,DFSUtil.string2Bytes("foo"),perm,0L);
  snapshottable.addSnapshottableFeature();
  final Snapshot[] snapshots={new Snapshot(1,"s1",snapshottable),new Snapshot(1,"s1",snapshottable),new Snapshot(2,"s2",snapshottable),new Snapshot(2,"s2",snapshottable)};
  Assert.assertEquals(0,Snapshot.ID_COMPARATOR.compare(null,null));
  for (  Snapshot s : snapshots) {
    Assert.assertTrue(Snapshot.ID_COMPARATOR.compare(null,s) > 0);
    Assert.assertTrue(Snapshot.ID_COMPARATOR.compare(s,null) < 0);
    for (    Snapshot t : snapshots) {
      final int expected=s.getRoot().getLocalName().compareTo(t.getRoot().getLocalName());
      final int computed=Snapshot.ID_COMPARATOR.compare(s,t);
      Assert.assertEquals(expected > 0,computed > 0);
      Assert.assertEquals(expected == 0,computed == 0);
      Assert.assertEquals(expected < 0,computed < 0);
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestRenameWithSnapshots

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier EqualityVerifier HybridVerifier 

@Test(timeout=300000) public void testRenameFromSDir2NonSDir() throws Exception {
  final String dirStr="/testRenameWithSnapshot";
  final String abcStr=dirStr + "/abc";
  final Path abc=new Path(abcStr);
  hdfs.mkdirs(abc,new FsPermission((short)0777));
  hdfs.allowSnapshot(abc);
  final Path foo=new Path(abc,"foo");
  DFSTestUtil.createFile(hdfs,foo,BLOCKSIZE,REPL,SEED);
  hdfs.createSnapshot(abc,"s0");
  try {
    hdfs.rename(abc,new Path(dirStr,"tmp"));
    fail("Expect exception since " + abc + " is snapshottable and already has snapshots");
  }
 catch (  IOException e) {
    GenericTestUtils.assertExceptionContains(abcStr + " is snapshottable and already has snapshots",e);
  }
  final String xyzStr=dirStr + "/xyz";
  final Path xyz=new Path(xyzStr);
  hdfs.mkdirs(xyz,new FsPermission((short)0777));
  final Path bar=new Path(xyz,"bar");
  hdfs.rename(foo,bar);
  final INode fooRef=fsdir.getINode(SnapshotTestHelper.getSnapshotPath(abc,"s0","foo").toString());
  Assert.assertTrue(fooRef.isReference());
  Assert.assertTrue(fooRef.asReference() instanceof INodeReference.WithName);
  final INodeReference.WithCount withCount=(INodeReference.WithCount)fooRef.asReference().getReferredINode();
  Assert.assertEquals(2,withCount.getReferenceCount());
  final INode barRef=fsdir.getINode(bar.toString());
  Assert.assertTrue(barRef.isReference());
  Assert.assertSame(withCount,barRef.asReference().getReferredINode());
  hdfs.delete(bar,false);
  Assert.assertEquals(1,withCount.getReferenceCount());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Make sure we clean the whole subtree under a DstReference node after 
 * deleting a snapshot.
 * see HDFS-5476.
 */
@Test public void testCleanDstReference() throws Exception {
  final Path test=new Path("/test");
  final Path foo=new Path(test,"foo");
  final Path bar=new Path(foo,"bar");
  hdfs.mkdirs(bar);
  SnapshotTestHelper.createSnapshot(hdfs,test,"s0");
  final Path fileInBar=new Path(bar,"file");
  DFSTestUtil.createFile(hdfs,fileInBar,BLOCKSIZE,REPL,SEED);
  final Path foo2=new Path(test,"foo2");
  hdfs.rename(foo,foo2);
  hdfs.createSnapshot(test,"s1");
  hdfs.delete(new Path(foo2,"bar"),true);
  hdfs.delete(foo2,true);
  final Path sfileInBar=SnapshotTestHelper.getSnapshotPath(test,"s1","foo2/bar/file");
  assertTrue(hdfs.exists(sfileInBar));
  hdfs.deleteSnapshot(test,"s1");
  assertFalse(hdfs.exists(sfileInBar));
  restartClusterAndCheckImage(true);
  final Path barInS0=SnapshotTestHelper.getSnapshotPath(test,"s0","foo/bar");
  INodeDirectory barNode=fsdir.getINode(barInS0.toString()).asDirectory();
  assertEquals(0,barNode.getChildrenList(Snapshot.CURRENT_STATE_ID).size());
  List diffList=barNode.getDiffs().asList();
  assertEquals(1,diffList.size());
  DirectoryDiff diff=diffList.get(0);
  assertEquals(0,diff.getChildrenDiff().getList(ListType.DELETED).size());
  assertEquals(0,diff.getChildrenDiff().getList(ListType.CREATED).size());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test the undo section of rename. Before the rename, we create the renamed 
 * file/dir before taking the snapshot.
 */
@Test public void testRenameUndo_1() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  final Path foo=new Path(sdir1,"foo");
  final Path bar=new Path(foo,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPL,SEED);
  final Path dir2file=new Path(sdir2,"file");
  DFSTestUtil.createFile(hdfs,dir2file,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  INodeDirectory dir2=fsdir.getINode4Write(sdir2.toString()).asDirectory();
  INodeDirectory mockDir2=spy(dir2);
  doReturn(false).when(mockDir2).addChild((INode)anyObject(),anyBoolean(),Mockito.anyInt());
  INodeDirectory root=fsdir.getINode4Write("/").asDirectory();
  root.replaceChild(dir2,mockDir2,fsdir.getINodeMap());
  final Path newfoo=new Path(sdir2,"foo");
  boolean result=hdfs.rename(foo,newfoo);
  assertFalse(result);
  INodeDirectory dir1Node=fsdir.getINode4Write(sdir1.toString()).asDirectory();
  Snapshot s1=dir1Node.getSnapshot(DFSUtil.string2Bytes("s1"));
  ReadOnlyList dir1Children=dir1Node.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(1,dir1Children.size());
  assertEquals(foo.getName(),dir1Children.get(0).getLocalName());
  List dir1Diffs=dir1Node.getDiffs().asList();
  assertEquals(1,dir1Diffs.size());
  assertEquals(s1.getId(),dir1Diffs.get(0).getSnapshotId());
  ChildrenDiff childrenDiff=dir1Diffs.get(0).getChildrenDiff();
  assertEquals(0,childrenDiff.getList(ListType.DELETED).size());
  assertEquals(0,childrenDiff.getList(ListType.CREATED).size());
  INode fooNode=fsdir.getINode4Write(foo.toString());
  assertTrue(fooNode.isDirectory() && fooNode.asDirectory().isWithSnapshot());
  List fooDiffs=fooNode.asDirectory().getDiffs().asList();
  assertEquals(1,fooDiffs.size());
  assertEquals(s1.getId(),fooDiffs.get(0).getSnapshotId());
  final Path foo_s1=SnapshotTestHelper.getSnapshotPath(sdir1,"s1","foo");
  INode fooNode_s1=fsdir.getINode(foo_s1.toString());
  assertTrue(fooNode_s1 == fooNode);
  assertFalse(hdfs.exists(newfoo));
  INodeDirectory dir2Node=fsdir.getINode4Write(sdir2.toString()).asDirectory();
  assertFalse(dir2Node.isWithSnapshot());
  ReadOnlyList dir2Children=dir2Node.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(1,dir2Children.size());
  assertEquals(dir2file.getName(),dir2Children.get(0).getLocalName());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier EqualityVerifier HybridVerifier 

/** 
 * Unit test for HDFS-4842.
 */
@Test public void testRenameDirAndDeleteSnapshot_7() throws Exception {
  fsn.getSnapshotManager().setAllowNestedSnapshots(true);
  final Path test=new Path("/test");
  final Path dir1=new Path(test,"dir1");
  final Path dir2=new Path(test,"dir2");
  hdfs.mkdirs(dir1);
  hdfs.mkdirs(dir2);
  final Path foo=new Path(dir2,"foo");
  final Path bar=new Path(foo,"bar");
  final Path file=new Path(bar,"file");
  DFSTestUtil.createFile(hdfs,file,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,test,"s0");
  SnapshotTestHelper.createSnapshot(hdfs,test,"s1");
  hdfs.delete(file,true);
  SnapshotTestHelper.createSnapshot(hdfs,dir2,"s2");
  final Path newfoo=new Path(dir1,foo.getName());
  hdfs.rename(foo,newfoo);
  hdfs.deleteSnapshot(test,"s1");
  final Path file_s2=SnapshotTestHelper.getSnapshotPath(dir2,"s2","foo/bar/file");
  assertFalse(hdfs.exists(file_s2));
  final Path file_s0=SnapshotTestHelper.getSnapshotPath(test,"s0","dir2/foo/bar/file");
  assertTrue(hdfs.exists(file_s0));
  INodeDirectory dir1Node=fsdir.getINode4Write(dir1.toString()).asDirectory();
  List dir1DiffList=dir1Node.getDiffs().asList();
  assertEquals(1,dir1DiffList.size());
  List dList=dir1DiffList.get(0).getChildrenDiff().getList(ListType.DELETED);
  assertTrue(dList.isEmpty());
  List cList=dir1DiffList.get(0).getChildrenDiff().getList(ListType.CREATED);
  assertEquals(1,cList.size());
  INode cNode=cList.get(0);
  INode fooNode=fsdir.getINode4Write(newfoo.toString());
  assertSame(cNode,fooNode);
  final Path newbar=new Path(newfoo,bar.getName());
  INodeDirectory barNode=fsdir.getINode4Write(newbar.toString()).asDirectory();
  assertSame(fooNode.asDirectory(),barNode.getParent());
  List barDiffList=barNode.getDiffs().asList();
  assertEquals(1,barDiffList.size());
  DirectoryDiff diff=barDiffList.get(0);
  INodeDirectory testNode=fsdir.getINode4Write(test.toString()).asDirectory();
  Snapshot s0=testNode.getSnapshot(DFSUtil.string2Bytes("s0"));
  assertEquals(s0.getId(),diff.getSnapshotId());
  assertEquals("file",diff.getChildrenDiff().getList(ListType.DELETED).get(0).getLocalName());
  INodeDirectory dir2Node=fsdir.getINode4Write(dir2.toString()).asDirectory();
  List dir2DiffList=dir2Node.getDiffs().asList();
  assertEquals(1,dir2DiffList.size());
  dList=dir2DiffList.get(0).getChildrenDiff().getList(ListType.DELETED);
  assertEquals(1,dList.size());
  final Path foo_s2=SnapshotTestHelper.getSnapshotPath(dir2,"s2",foo.getName());
  INodeReference.WithName fooNode_s2=(INodeReference.WithName)fsdir.getINode(foo_s2.toString());
  assertSame(dList.get(0),fooNode_s2);
  assertSame(fooNode.asReference().getReferredINode(),fooNode_s2.getReferredINode());
  restartClusterAndCheckImage(true);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test the undo section of rename. Before the rename, we create the renamed 
 * file/dir after taking the snapshot.
 */
@Test public void testRenameUndo_2() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  final Path dir2file=new Path(sdir2,"file");
  DFSTestUtil.createFile(hdfs,dir2file,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  final Path foo=new Path(sdir1,"foo");
  final Path bar=new Path(foo,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPL,SEED);
  INodeDirectory dir2=fsdir.getINode4Write(sdir2.toString()).asDirectory();
  INodeDirectory mockDir2=spy(dir2);
  doReturn(false).when(mockDir2).addChild((INode)anyObject(),anyBoolean(),Mockito.anyInt());
  INodeDirectory root=fsdir.getINode4Write("/").asDirectory();
  root.replaceChild(dir2,mockDir2,fsdir.getINodeMap());
  final Path newfoo=new Path(sdir2,"foo");
  boolean result=hdfs.rename(foo,newfoo);
  assertFalse(result);
  INodeDirectory dir1Node=fsdir.getINode4Write(sdir1.toString()).asDirectory();
  Snapshot s1=dir1Node.getSnapshot(DFSUtil.string2Bytes("s1"));
  ReadOnlyList dir1Children=dir1Node.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(1,dir1Children.size());
  assertEquals(foo.getName(),dir1Children.get(0).getLocalName());
  List dir1Diffs=dir1Node.getDiffs().asList();
  assertEquals(1,dir1Diffs.size());
  assertEquals(s1.getId(),dir1Diffs.get(0).getSnapshotId());
  ChildrenDiff childrenDiff=dir1Diffs.get(0).getChildrenDiff();
  assertEquals(0,childrenDiff.getList(ListType.DELETED).size());
  assertEquals(1,childrenDiff.getList(ListType.CREATED).size());
  INode fooNode=fsdir.getINode4Write(foo.toString());
  assertTrue(fooNode instanceof INodeDirectory);
  assertTrue(childrenDiff.getList(ListType.CREATED).get(0) == fooNode);
  final Path foo_s1=SnapshotTestHelper.getSnapshotPath(sdir1,"s1","foo");
  assertFalse(hdfs.exists(foo_s1));
  assertFalse(hdfs.exists(newfoo));
  INodeDirectory dir2Node=fsdir.getINode4Write(sdir2.toString()).asDirectory();
  assertFalse(dir2Node.isWithSnapshot());
  ReadOnlyList dir2Children=dir2Node.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(1,dir2Children.size());
  assertEquals(dir2file.getName(),dir2Children.get(0).getLocalName());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test rename a dir and a file multiple times across snapshottable 
 * directories: /dir1/foo -> /dir2/foo -> /dir3/foo -> /dir2/foo -> /dir1/foo
 * Only create snapshots in the beginning (before the rename).
 */
@Test public void testRenameMoreThanOnceAcrossSnapDirs() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  final Path sdir3=new Path("/dir3");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  hdfs.mkdirs(sdir3);
  final Path foo_dir1=new Path(sdir1,"foo");
  final Path bar1_dir1=new Path(foo_dir1,"bar1");
  final Path bar2_dir1=new Path(sdir1,"bar");
  DFSTestUtil.createFile(hdfs,bar1_dir1,BLOCKSIZE,REPL,SEED);
  DFSTestUtil.createFile(hdfs,bar2_dir1,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2");
  SnapshotTestHelper.createSnapshot(hdfs,sdir3,"s3");
  final Path foo_dir2=new Path(sdir2,"foo");
  hdfs.rename(foo_dir1,foo_dir2);
  final Path bar2_dir2=new Path(sdir2,"bar");
  hdfs.rename(bar2_dir1,bar2_dir2);
  restartClusterAndCheckImage(true);
  final Path bar1_dir2=new Path(foo_dir2,"bar1");
  hdfs.setReplication(bar1_dir2,REPL_1);
  hdfs.setReplication(bar2_dir2,REPL_1);
  final Path bar1_s1=SnapshotTestHelper.getSnapshotPath(sdir1,"s1","foo/bar1");
  final Path bar2_s1=SnapshotTestHelper.getSnapshotPath(sdir1,"s1","bar");
  final Path bar1_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo/bar1");
  final Path bar2_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","bar");
  assertTrue(hdfs.exists(bar1_s1));
  assertTrue(hdfs.exists(bar2_s1));
  assertFalse(hdfs.exists(bar1_s2));
  assertFalse(hdfs.exists(bar2_s2));
  FileStatus statusBar1=hdfs.getFileStatus(bar1_s1);
  assertEquals(REPL,statusBar1.getReplication());
  statusBar1=hdfs.getFileStatus(bar1_dir2);
  assertEquals(REPL_1,statusBar1.getReplication());
  FileStatus statusBar2=hdfs.getFileStatus(bar2_s1);
  assertEquals(REPL,statusBar2.getReplication());
  statusBar2=hdfs.getFileStatus(bar2_dir2);
  assertEquals(REPL_1,statusBar2.getReplication());
  final Path foo_dir3=new Path(sdir3,"foo");
  hdfs.rename(foo_dir2,foo_dir3);
  final Path bar2_dir3=new Path(sdir3,"bar");
  hdfs.rename(bar2_dir2,bar2_dir3);
  restartClusterAndCheckImage(true);
  final Path bar1_dir3=new Path(foo_dir3,"bar1");
  hdfs.setReplication(bar1_dir3,REPL_2);
  hdfs.setReplication(bar2_dir3,REPL_2);
  final Path bar1_s3=SnapshotTestHelper.getSnapshotPath(sdir3,"s3","foo/bar1");
  final Path bar2_s3=SnapshotTestHelper.getSnapshotPath(sdir3,"s3","bar");
  assertTrue(hdfs.exists(bar1_s1));
  assertTrue(hdfs.exists(bar2_s1));
  assertFalse(hdfs.exists(bar1_s2));
  assertFalse(hdfs.exists(bar2_s2));
  assertFalse(hdfs.exists(bar1_s3));
  assertFalse(hdfs.exists(bar2_s3));
  statusBar1=hdfs.getFileStatus(bar1_s1);
  assertEquals(REPL,statusBar1.getReplication());
  statusBar1=hdfs.getFileStatus(bar1_dir3);
  assertEquals(REPL_2,statusBar1.getReplication());
  statusBar2=hdfs.getFileStatus(bar2_s1);
  assertEquals(REPL,statusBar2.getReplication());
  statusBar2=hdfs.getFileStatus(bar2_dir3);
  assertEquals(REPL_2,statusBar2.getReplication());
  hdfs.rename(foo_dir3,foo_dir2);
  hdfs.rename(bar2_dir3,bar2_dir2);
  restartClusterAndCheckImage(true);
  hdfs.setReplication(bar1_dir2,REPL);
  hdfs.setReplication(bar2_dir2,REPL);
  assertTrue(hdfs.exists(bar1_s1));
  assertTrue(hdfs.exists(bar2_s1));
  assertFalse(hdfs.exists(bar1_s2));
  assertFalse(hdfs.exists(bar2_s2));
  assertFalse(hdfs.exists(bar1_s3));
  assertFalse(hdfs.exists(bar2_s3));
  statusBar1=hdfs.getFileStatus(bar1_s1);
  assertEquals(REPL,statusBar1.getReplication());
  statusBar1=hdfs.getFileStatus(bar1_dir2);
  assertEquals(REPL,statusBar1.getReplication());
  statusBar2=hdfs.getFileStatus(bar2_s1);
  assertEquals(REPL,statusBar2.getReplication());
  statusBar2=hdfs.getFileStatus(bar2_dir2);
  assertEquals(REPL,statusBar2.getReplication());
  hdfs.rename(foo_dir2,foo_dir1);
  hdfs.rename(bar2_dir2,bar2_dir1);
  INodeReference fooRef=fsdir.getINode4Write(foo_dir1.toString()).asReference();
  INodeReference.WithCount fooWithCount=(WithCount)fooRef.getReferredINode();
  assertEquals(2,fooWithCount.getReferenceCount());
  INodeDirectory foo=fooWithCount.asDirectory();
  assertEquals(1,foo.getDiffs().asList().size());
  INodeDirectory sdir1Node=fsdir.getINode(sdir1.toString()).asDirectory();
  Snapshot s1=sdir1Node.getSnapshot(DFSUtil.string2Bytes("s1"));
  assertEquals(s1.getId(),foo.getDirectoryWithSnapshotFeature().getLastSnapshotId());
  INodeFile bar1=fsdir.getINode4Write(bar1_dir1.toString()).asFile();
  assertEquals(1,bar1.getDiffs().asList().size());
  assertEquals(s1.getId(),bar1.getDiffs().getLastSnapshotId());
  INodeReference barRef=fsdir.getINode4Write(bar2_dir1.toString()).asReference();
  INodeReference.WithCount barWithCount=(WithCount)barRef.getReferredINode();
  assertEquals(2,barWithCount.getReferenceCount());
  INodeFile bar=barWithCount.asFile();
  assertEquals(1,bar.getDiffs().asList().size());
  assertEquals(s1.getId(),bar.getDiffs().getLastSnapshotId());
  restartClusterAndCheckImage(true);
  hdfs.delete(foo_dir1,true);
  hdfs.delete(bar2_dir1,true);
  restartClusterAndCheckImage(true);
  assertTrue(hdfs.exists(bar1_s1));
  assertTrue(hdfs.exists(bar2_s1));
  assertFalse(hdfs.exists(bar1_s2));
  assertFalse(hdfs.exists(bar2_s2));
  assertFalse(hdfs.exists(bar1_s3));
  assertFalse(hdfs.exists(bar2_s3));
  assertFalse(hdfs.exists(foo_dir1));
  assertFalse(hdfs.exists(bar1_dir1));
  assertFalse(hdfs.exists(bar2_dir1));
  statusBar1=hdfs.getFileStatus(bar1_s1);
  assertEquals(REPL,statusBar1.getReplication());
  statusBar2=hdfs.getFileStatus(bar2_s1);
  assertEquals(REPL,statusBar2.getReplication());
  final Path foo_s1=SnapshotTestHelper.getSnapshotPath(sdir1,"s1","foo");
  fooRef=fsdir.getINode(foo_s1.toString()).asReference();
  fooWithCount=(WithCount)fooRef.getReferredINode();
  assertEquals(1,fooWithCount.getReferenceCount());
  barRef=fsdir.getINode(bar2_s1.toString()).asReference();
  barWithCount=(WithCount)barRef.getReferredINode();
  assertEquals(1,barWithCount.getReferenceCount());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test the undo section of the second-time rename.
 */
@Test public void testRenameUndo_3() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  final Path sdir3=new Path("/dir3");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  hdfs.mkdirs(sdir3);
  final Path foo=new Path(sdir1,"foo");
  final Path bar=new Path(foo,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2");
  INodeDirectory dir3=fsdir.getINode4Write(sdir3.toString()).asDirectory();
  INodeDirectory mockDir3=spy(dir3);
  doReturn(false).when(mockDir3).addChild((INode)anyObject(),anyBoolean(),Mockito.anyInt());
  INodeDirectory root=fsdir.getINode4Write("/").asDirectory();
  root.replaceChild(dir3,mockDir3,fsdir.getINodeMap());
  final Path foo_dir2=new Path(sdir2,"foo2");
  final Path foo_dir3=new Path(sdir3,"foo3");
  hdfs.rename(foo,foo_dir2);
  boolean result=hdfs.rename(foo_dir2,foo_dir3);
  assertFalse(result);
  INodeDirectory dir1Node=fsdir.getINode4Write(sdir1.toString()).asDirectory();
  Snapshot s1=dir1Node.getSnapshot(DFSUtil.string2Bytes("s1"));
  INodeDirectory dir2Node=fsdir.getINode4Write(sdir2.toString()).asDirectory();
  Snapshot s2=dir2Node.getSnapshot(DFSUtil.string2Bytes("s2"));
  ReadOnlyList dir2Children=dir2Node.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(1,dir2Children.size());
  List dir2Diffs=dir2Node.getDiffs().asList();
  assertEquals(1,dir2Diffs.size());
  assertEquals(s2.getId(),dir2Diffs.get(0).getSnapshotId());
  ChildrenDiff childrenDiff=dir2Diffs.get(0).getChildrenDiff();
  assertEquals(0,childrenDiff.getList(ListType.DELETED).size());
  assertEquals(1,childrenDiff.getList(ListType.CREATED).size());
  final Path foo_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo2");
  assertFalse(hdfs.exists(foo_s2));
  INode fooNode=fsdir.getINode4Write(foo_dir2.toString());
  assertTrue(childrenDiff.getList(ListType.CREATED).get(0) == fooNode);
  assertTrue(fooNode instanceof INodeReference.DstReference);
  List fooDiffs=fooNode.asDirectory().getDiffs().asList();
  assertEquals(1,fooDiffs.size());
  assertEquals(s1.getId(),fooDiffs.get(0).getSnapshotId());
  hdfs.createSnapshot(sdir2,"s3");
  result=hdfs.rename(foo_dir2,foo_dir3);
  assertFalse(result);
  dir2Node=fsdir.getINode4Write(sdir2.toString()).asDirectory();
  Snapshot s3=dir2Node.getSnapshot(DFSUtil.string2Bytes("s3"));
  fooNode=fsdir.getINode4Write(foo_dir2.toString());
  dir2Children=dir2Node.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(1,dir2Children.size());
  dir2Diffs=dir2Node.getDiffs().asList();
  assertEquals(2,dir2Diffs.size());
  assertEquals(s2.getId(),dir2Diffs.get(0).getSnapshotId());
  assertEquals(s3.getId(),dir2Diffs.get(1).getSnapshotId());
  childrenDiff=dir2Diffs.get(0).getChildrenDiff();
  assertEquals(0,childrenDiff.getList(ListType.DELETED).size());
  assertEquals(1,childrenDiff.getList(ListType.CREATED).size());
  assertTrue(childrenDiff.getList(ListType.CREATED).get(0) == fooNode);
  childrenDiff=dir2Diffs.get(1).getChildrenDiff();
  assertEquals(0,childrenDiff.getList(ListType.DELETED).size());
  assertEquals(0,childrenDiff.getList(ListType.CREATED).size());
  final Path foo_s3=SnapshotTestHelper.getSnapshotPath(sdir2,"s3","foo2");
  assertFalse(hdfs.exists(foo_s2));
  assertTrue(hdfs.exists(foo_s3));
  assertTrue(fooNode instanceof INodeReference.DstReference);
  fooDiffs=fooNode.asDirectory().getDiffs().asList();
  assertEquals(2,fooDiffs.size());
  assertEquals(s1.getId(),fooDiffs.get(0).getSnapshotId());
  assertEquals(s3.getId(),fooDiffs.get(1).getSnapshotId());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier EqualityVerifier HybridVerifier 

/** 
 * After the following operations:
 * Rename a dir -> create a snapshot s on dst tree -> rename the renamed dir
 * again -> delete snapshot s on dst tree
 * Make sure we only delete the snapshot s under the renamed dir.
 */
@Test public void testRenameDirAndDeleteSnapshot_4() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  final Path foo=new Path(sdir1,"foo");
  final Path bar=new Path(foo,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPL,SEED);
  hdfs.mkdirs(sdir2);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2");
  final Path foo2=new Path(sdir2,"foo");
  hdfs.rename(foo,foo2);
  final Path bar2=new Path(foo2,"bar2");
  DFSTestUtil.createFile(hdfs,bar2,BLOCKSIZE,REPL,SEED);
  final Path bar3=new Path(foo2,"bar3");
  DFSTestUtil.createFile(hdfs,bar3,BLOCKSIZE,REPL,SEED);
  hdfs.createSnapshot(sdir2,"s3");
  hdfs.rename(foo2,foo);
  hdfs.deleteSnapshot(sdir2,"s3");
  final INodeDirectory dir1Node=fsdir.getINode4Write(sdir1.toString()).asDirectory();
  Quota.Counts q1=dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
  assertEquals(9,q1.get(Quota.NAMESPACE));
  final INodeDirectory dir2Node=fsdir.getINode4Write(sdir2.toString()).asDirectory();
  Quota.Counts q2=dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
  assertEquals(2,q2.get(Quota.NAMESPACE));
  final Path foo_s1=SnapshotTestHelper.getSnapshotPath(sdir1,"s1",foo.getName());
  final INode fooRef=fsdir.getINode(foo_s1.toString());
  assertTrue(fooRef instanceof INodeReference.WithName);
  INodeReference.WithCount wc=(WithCount)fooRef.asReference().getReferredINode();
  assertEquals(2,wc.getReferenceCount());
  INodeDirectory fooNode=wc.getReferredINode().asDirectory();
  ReadOnlyList children=fooNode.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(3,children.size());
  assertEquals(bar.getName(),children.get(0).getLocalName());
  assertEquals(bar2.getName(),children.get(1).getLocalName());
  assertEquals(bar3.getName(),children.get(2).getLocalName());
  List diffList=fooNode.getDiffs().asList();
  assertEquals(1,diffList.size());
  Snapshot s1=dir1Node.getSnapshot(DFSUtil.string2Bytes("s1"));
  assertEquals(s1.getId(),diffList.get(0).getSnapshotId());
  ChildrenDiff diff=diffList.get(0).getChildrenDiff();
  assertEquals(2,diff.getList(ListType.CREATED).size());
  assertEquals(0,diff.getList(ListType.DELETED).size());
  final INode fooRef2=fsdir.getINode4Write(foo.toString());
  assertTrue(fooRef2 instanceof INodeReference.DstReference);
  INodeReference.WithCount wc2=(WithCount)fooRef2.asReference().getReferredINode();
  assertSame(wc,wc2);
  assertSame(fooRef2,wc.getParentReference());
  restartClusterAndCheckImage(true);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test rename a dir multiple times across snapshottable directories: 
 * /dir1/foo -> /dir2/foo -> /dir3/foo -> /dir2/foo -> /dir1/foo
 * Create snapshots after each rename.
 */
@Test public void testRenameMoreThanOnceAcrossSnapDirs_2() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  final Path sdir3=new Path("/dir3");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  hdfs.mkdirs(sdir3);
  final Path foo_dir1=new Path(sdir1,"foo");
  final Path bar1_dir1=new Path(foo_dir1,"bar1");
  final Path bar_dir1=new Path(sdir1,"bar");
  DFSTestUtil.createFile(hdfs,bar1_dir1,BLOCKSIZE,REPL,SEED);
  DFSTestUtil.createFile(hdfs,bar_dir1,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2");
  SnapshotTestHelper.createSnapshot(hdfs,sdir3,"s3");
  final Path foo_dir2=new Path(sdir2,"foo");
  hdfs.rename(foo_dir1,foo_dir2);
  final Path bar_dir2=new Path(sdir2,"bar");
  hdfs.rename(bar_dir1,bar_dir2);
  final Path bar1_dir2=new Path(foo_dir2,"bar1");
  hdfs.setReplication(bar1_dir2,REPL_1);
  hdfs.setReplication(bar_dir2,REPL_1);
  restartClusterAndCheckImage(true);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s11");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s22");
  SnapshotTestHelper.createSnapshot(hdfs,sdir3,"s33");
  final Path foo_dir3=new Path(sdir3,"foo");
  hdfs.rename(foo_dir2,foo_dir3);
  final Path bar_dir3=new Path(sdir3,"bar");
  hdfs.rename(bar_dir2,bar_dir3);
  final Path bar1_dir3=new Path(foo_dir3,"bar1");
  hdfs.setReplication(bar1_dir3,REPL_2);
  hdfs.setReplication(bar_dir3,REPL_2);
  restartClusterAndCheckImage(true);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s111");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s222");
  SnapshotTestHelper.createSnapshot(hdfs,sdir3,"s333");
  final Path bar1_s1=SnapshotTestHelper.getSnapshotPath(sdir1,"s1","foo/bar1");
  final Path bar1_s22=SnapshotTestHelper.getSnapshotPath(sdir2,"s22","foo/bar1");
  final Path bar1_s333=SnapshotTestHelper.getSnapshotPath(sdir3,"s333","foo/bar1");
  final Path bar_s1=SnapshotTestHelper.getSnapshotPath(sdir1,"s1","bar");
  final Path bar_s22=SnapshotTestHelper.getSnapshotPath(sdir2,"s22","bar");
  final Path bar_s333=SnapshotTestHelper.getSnapshotPath(sdir3,"s333","bar");
  assertTrue(hdfs.exists(bar1_s1));
  assertTrue(hdfs.exists(bar1_s22));
  assertTrue(hdfs.exists(bar1_s333));
  assertTrue(hdfs.exists(bar_s1));
  assertTrue(hdfs.exists(bar_s22));
  assertTrue(hdfs.exists(bar_s333));
  FileStatus statusBar1=hdfs.getFileStatus(bar1_s1);
  assertEquals(REPL,statusBar1.getReplication());
  statusBar1=hdfs.getFileStatus(bar1_dir3);
  assertEquals(REPL_2,statusBar1.getReplication());
  statusBar1=hdfs.getFileStatus(bar1_s22);
  assertEquals(REPL_1,statusBar1.getReplication());
  statusBar1=hdfs.getFileStatus(bar1_s333);
  assertEquals(REPL_2,statusBar1.getReplication());
  FileStatus statusBar=hdfs.getFileStatus(bar_s1);
  assertEquals(REPL,statusBar.getReplication());
  statusBar=hdfs.getFileStatus(bar_dir3);
  assertEquals(REPL_2,statusBar.getReplication());
  statusBar=hdfs.getFileStatus(bar_s22);
  assertEquals(REPL_1,statusBar.getReplication());
  statusBar=hdfs.getFileStatus(bar_s333);
  assertEquals(REPL_2,statusBar.getReplication());
  hdfs.rename(foo_dir3,foo_dir2);
  hdfs.rename(bar_dir3,bar_dir2);
  hdfs.setReplication(bar1_dir2,REPL);
  hdfs.setReplication(bar_dir2,REPL);
  restartClusterAndCheckImage(true);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1111");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2222");
  final Path bar1_s2222=SnapshotTestHelper.getSnapshotPath(sdir2,"s2222","foo/bar1");
  final Path bar_s2222=SnapshotTestHelper.getSnapshotPath(sdir2,"s2222","bar");
  assertTrue(hdfs.exists(bar1_s1));
  assertTrue(hdfs.exists(bar1_s22));
  assertTrue(hdfs.exists(bar1_s333));
  assertTrue(hdfs.exists(bar1_s2222));
  assertTrue(hdfs.exists(bar_s1));
  assertTrue(hdfs.exists(bar_s22));
  assertTrue(hdfs.exists(bar_s333));
  assertTrue(hdfs.exists(bar_s2222));
  statusBar1=hdfs.getFileStatus(bar1_s1);
  assertEquals(REPL,statusBar1.getReplication());
  statusBar1=hdfs.getFileStatus(bar1_dir2);
  assertEquals(REPL,statusBar1.getReplication());
  statusBar1=hdfs.getFileStatus(bar1_s22);
  assertEquals(REPL_1,statusBar1.getReplication());
  statusBar1=hdfs.getFileStatus(bar1_s333);
  assertEquals(REPL_2,statusBar1.getReplication());
  statusBar1=hdfs.getFileStatus(bar1_s2222);
  assertEquals(REPL,statusBar1.getReplication());
  statusBar=hdfs.getFileStatus(bar_s1);
  assertEquals(REPL,statusBar.getReplication());
  statusBar=hdfs.getFileStatus(bar_dir2);
  assertEquals(REPL,statusBar.getReplication());
  statusBar=hdfs.getFileStatus(bar_s22);
  assertEquals(REPL_1,statusBar.getReplication());
  statusBar=hdfs.getFileStatus(bar_s333);
  assertEquals(REPL_2,statusBar.getReplication());
  statusBar=hdfs.getFileStatus(bar_s2222);
  assertEquals(REPL,statusBar.getReplication());
  hdfs.rename(foo_dir2,foo_dir1);
  hdfs.rename(bar_dir2,bar_dir1);
  INodeDirectory sdir1Node=fsdir.getINode(sdir1.toString()).asDirectory();
  INodeDirectory sdir2Node=fsdir.getINode(sdir2.toString()).asDirectory();
  INodeDirectory sdir3Node=fsdir.getINode(sdir3.toString()).asDirectory();
  INodeReference fooRef=fsdir.getINode4Write(foo_dir1.toString()).asReference();
  INodeReference.WithCount fooWithCount=(WithCount)fooRef.getReferredINode();
  assertEquals(5,fooWithCount.getReferenceCount());
  INodeDirectory foo=fooWithCount.asDirectory();
  List fooDiffs=foo.getDiffs().asList();
  assertEquals(4,fooDiffs.size());
  Snapshot s2222=sdir2Node.getSnapshot(DFSUtil.string2Bytes("s2222"));
  Snapshot s333=sdir3Node.getSnapshot(DFSUtil.string2Bytes("s333"));
  Snapshot s22=sdir2Node.getSnapshot(DFSUtil.string2Bytes("s22"));
  Snapshot s1=sdir1Node.getSnapshot(DFSUtil.string2Bytes("s1"));
  assertEquals(s2222.getId(),fooDiffs.get(3).getSnapshotId());
  assertEquals(s333.getId(),fooDiffs.get(2).getSnapshotId());
  assertEquals(s22.getId(),fooDiffs.get(1).getSnapshotId());
  assertEquals(s1.getId(),fooDiffs.get(0).getSnapshotId());
  INodeFile bar1=fsdir.getINode4Write(bar1_dir1.toString()).asFile();
  List bar1Diffs=bar1.getDiffs().asList();
  assertEquals(3,bar1Diffs.size());
  assertEquals(s333.getId(),bar1Diffs.get(2).getSnapshotId());
  assertEquals(s22.getId(),bar1Diffs.get(1).getSnapshotId());
  assertEquals(s1.getId(),bar1Diffs.get(0).getSnapshotId());
  INodeReference barRef=fsdir.getINode4Write(bar_dir1.toString()).asReference();
  INodeReference.WithCount barWithCount=(WithCount)barRef.getReferredINode();
  assertEquals(5,barWithCount.getReferenceCount());
  INodeFile bar=barWithCount.asFile();
  List barDiffs=bar.getDiffs().asList();
  assertEquals(4,barDiffs.size());
  assertEquals(s2222.getId(),barDiffs.get(3).getSnapshotId());
  assertEquals(s333.getId(),barDiffs.get(2).getSnapshotId());
  assertEquals(s22.getId(),barDiffs.get(1).getSnapshotId());
  assertEquals(s1.getId(),barDiffs.get(0).getSnapshotId());
  restartClusterAndCheckImage(true);
  hdfs.delete(foo_dir1,true);
  hdfs.delete(bar_dir1,true);
  restartClusterAndCheckImage(true);
  final Path bar1_s1111=SnapshotTestHelper.getSnapshotPath(sdir1,"s1111","foo/bar1");
  final Path bar_s1111=SnapshotTestHelper.getSnapshotPath(sdir1,"s1111","bar");
  assertTrue(hdfs.exists(bar1_s1));
  assertTrue(hdfs.exists(bar1_s22));
  assertTrue(hdfs.exists(bar1_s333));
  assertTrue(hdfs.exists(bar1_s2222));
  assertFalse(hdfs.exists(bar1_s1111));
  assertTrue(hdfs.exists(bar_s1));
  assertTrue(hdfs.exists(bar_s22));
  assertTrue(hdfs.exists(bar_s333));
  assertTrue(hdfs.exists(bar_s2222));
  assertFalse(hdfs.exists(bar_s1111));
  final Path foo_s2222=SnapshotTestHelper.getSnapshotPath(sdir2,"s2222","foo");
  fooRef=fsdir.getINode(foo_s2222.toString()).asReference();
  fooWithCount=(WithCount)fooRef.getReferredINode();
  assertEquals(4,fooWithCount.getReferenceCount());
  foo=fooWithCount.asDirectory();
  fooDiffs=foo.getDiffs().asList();
  assertEquals(4,fooDiffs.size());
  assertEquals(s2222.getId(),fooDiffs.get(3).getSnapshotId());
  bar1Diffs=bar1.getDiffs().asList();
  assertEquals(3,bar1Diffs.size());
  assertEquals(s333.getId(),bar1Diffs.get(2).getSnapshotId());
  barRef=fsdir.getINode(bar_s2222.toString()).asReference();
  barWithCount=(WithCount)barRef.getReferredINode();
  assertEquals(4,barWithCount.getReferenceCount());
  bar=barWithCount.asFile();
  barDiffs=bar.getDiffs().asList();
  assertEquals(4,barDiffs.size());
  assertEquals(s2222.getId(),barDiffs.get(3).getSnapshotId());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier EqualityVerifier HybridVerifier 

/** 
 * Test rename while the rename operation will exceed the quota in the dst
 * tree.
 */
@Test public void testRenameUndo_5() throws Exception {
  final Path test=new Path("/test");
  final Path dir1=new Path(test,"dir1");
  final Path dir2=new Path(test,"dir2");
  final Path subdir2=new Path(dir2,"subdir2");
  hdfs.mkdirs(dir1);
  hdfs.mkdirs(subdir2);
  final Path foo=new Path(dir1,"foo");
  final Path bar=new Path(foo,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,dir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,dir2,"s2");
  hdfs.setQuota(dir2,5,Long.MAX_VALUE - 1);
  final Path foo2=new Path(subdir2,foo.getName());
  boolean rename=hdfs.rename(foo,foo2);
  assertFalse(rename);
  assertTrue(hdfs.exists(foo));
  assertTrue(hdfs.exists(bar));
  INodeDirectory dir1Node=fsdir.getINode4Write(dir1.toString()).asDirectory();
  List childrenList=ReadOnlyList.Util.asList(dir1Node.getChildrenList(Snapshot.CURRENT_STATE_ID));
  assertEquals(1,childrenList.size());
  INode fooNode=childrenList.get(0);
  assertTrue(fooNode.asDirectory().isWithSnapshot());
  INode barNode=fsdir.getINode4Write(bar.toString());
  assertTrue(barNode.getClass() == INodeFile.class);
  assertSame(fooNode,barNode.getParent());
  List diffList=dir1Node.getDiffs().asList();
  assertEquals(1,diffList.size());
  DirectoryDiff diff=diffList.get(0);
  assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
  assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
  INodeDirectory dir2Node=fsdir.getINode4Write(dir2.toString()).asDirectory();
  assertTrue(dir2Node.isSnapshottable());
  Quota.Counts counts=dir2Node.computeQuotaUsage();
  assertEquals(3,counts.get(Quota.NAMESPACE));
  assertEquals(0,counts.get(Quota.DISKSPACE));
  childrenList=ReadOnlyList.Util.asList(dir2Node.asDirectory().getChildrenList(Snapshot.CURRENT_STATE_ID));
  assertEquals(1,childrenList.size());
  INode subdir2Node=childrenList.get(0);
  assertSame(dir2Node,subdir2Node.getParent());
  assertSame(subdir2Node,fsdir.getINode4Write(subdir2.toString()));
  diffList=dir2Node.getDiffs().asList();
  assertEquals(1,diffList.size());
  diff=diffList.get(0);
  assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
  assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * After the following operations:
 * Rename a dir -> create a snapshot s on dst tree -> delete the renamed dir
 * -> delete snapshot s on dst tree
 * Make sure we destroy everything created after the rename under the renamed
 * dir.
 */
@Test public void testRenameDirAndDeleteSnapshot_3() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  final Path foo=new Path(sdir1,"foo");
  final Path bar=new Path(foo,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPL,SEED);
  hdfs.mkdirs(sdir2);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2");
  final Path foo2=new Path(sdir2,"foo");
  hdfs.rename(foo,foo2);
  final Path bar2=new Path(foo2,"bar2");
  DFSTestUtil.createFile(hdfs,bar2,BLOCKSIZE,REPL,SEED);
  final Path bar3=new Path(foo2,"bar3");
  DFSTestUtil.createFile(hdfs,bar3,BLOCKSIZE,REPL,SEED);
  hdfs.createSnapshot(sdir2,"s3");
  hdfs.delete(foo2,true);
  hdfs.deleteSnapshot(sdir2,"s3");
  final INodeDirectory dir1Node=fsdir.getINode4Write(sdir1.toString()).asDirectory();
  Quota.Counts q1=dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
  assertEquals(4,q1.get(Quota.NAMESPACE));
  final INodeDirectory dir2Node=fsdir.getINode4Write(sdir2.toString()).asDirectory();
  Quota.Counts q2=dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
  assertEquals(2,q2.get(Quota.NAMESPACE));
  final Path foo_s1=SnapshotTestHelper.getSnapshotPath(sdir1,"s1",foo.getName());
  INode fooRef=fsdir.getINode(foo_s1.toString());
  assertTrue(fooRef instanceof INodeReference.WithName);
  INodeReference.WithCount wc=(WithCount)fooRef.asReference().getReferredINode();
  assertEquals(1,wc.getReferenceCount());
  INodeDirectory fooNode=wc.getReferredINode().asDirectory();
  ReadOnlyList children=fooNode.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(1,children.size());
  assertEquals(bar.getName(),children.get(0).getLocalName());
  List diffList=fooNode.getDiffs().asList();
  assertEquals(1,diffList.size());
  Snapshot s1=dir1Node.getSnapshot(DFSUtil.string2Bytes("s1"));
  assertEquals(s1.getId(),diffList.get(0).getSnapshotId());
  ChildrenDiff diff=diffList.get(0).getChildrenDiff();
  assertEquals(0,diff.getList(ListType.CREATED).size());
  assertEquals(0,diff.getList(ListType.DELETED).size());
  restartClusterAndCheckImage(true);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Rename and deletion snapshot under the same the snapshottable directory.
 */
@Test public void testRenameDirAndDeleteSnapshot_6() throws Exception {
  final Path test=new Path("/test");
  final Path dir1=new Path(test,"dir1");
  final Path dir2=new Path(test,"dir2");
  hdfs.mkdirs(dir1);
  hdfs.mkdirs(dir2);
  final Path foo=new Path(dir2,"foo");
  final Path bar=new Path(foo,"bar");
  final Path file=new Path(bar,"file");
  DFSTestUtil.createFile(hdfs,file,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,test,"s0");
  hdfs.delete(file,true);
  final Path newfoo=new Path(dir1,foo.getName());
  hdfs.rename(foo,newfoo);
  final Path foo_s0=SnapshotTestHelper.getSnapshotPath(test,"s0","dir2/foo");
  assertTrue("the snapshot path " + foo_s0 + " should exist",hdfs.exists(foo_s0));
  hdfs.deleteSnapshot(test,"s0");
  assertFalse("after deleting s0, " + foo_s0 + " should not exist",hdfs.exists(foo_s0));
  INodeDirectory dir2Node=fsdir.getINode4Write(dir2.toString()).asDirectory();
  assertTrue("the diff list of " + dir2 + " should be empty after deleting s0",dir2Node.getDiffs().asList().isEmpty());
  assertTrue(hdfs.exists(newfoo));
  INode fooRefNode=fsdir.getINode4Write(newfoo.toString());
  assertTrue(fooRefNode instanceof INodeReference.DstReference);
  INodeDirectory fooNode=fooRefNode.asDirectory();
  assertTrue(fooNode.isWithSnapshot());
  assertTrue(fooNode.getDiffs().asList().isEmpty());
  INodeDirectory barNode=fooNode.getChildrenList(Snapshot.CURRENT_STATE_ID).get(0).asDirectory();
  assertTrue(barNode.getDiffs().asList().isEmpty());
  assertTrue(barNode.getChildrenList(Snapshot.CURRENT_STATE_ID).isEmpty());
  restartClusterAndCheckImage(true);
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier EqualityVerifier HybridVerifier 

/** 
 * Test the rename undo when removing dst node fails
 */
@Test public void testRenameUndo_6() throws Exception {
  final Path test=new Path("/test");
  final Path dir1=new Path(test,"dir1");
  final Path dir2=new Path(test,"dir2");
  final Path sub_dir2=new Path(dir2,"subdir");
  final Path subsub_dir2=new Path(sub_dir2,"subdir");
  hdfs.mkdirs(dir1);
  hdfs.mkdirs(subsub_dir2);
  final Path foo=new Path(dir1,"foo");
  hdfs.mkdirs(foo);
  SnapshotTestHelper.createSnapshot(hdfs,dir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,dir2,"s2");
  hdfs.setQuota(dir2,4,Long.MAX_VALUE - 1);
  try {
    hdfs.rename(foo,subsub_dir2,Rename.OVERWRITE);
    fail("Expect QuotaExceedException");
  }
 catch (  QuotaExceededException e) {
    String msg="Failed to record modification for snapshot: " + "The NameSpace quota (directories and files)" + " is exceeded: quota=4 file count=5";
    GenericTestUtils.assertExceptionContains(msg,e);
  }
  assertTrue(hdfs.exists(foo));
  INodeDirectory dir1Node=fsdir.getINode4Write(dir1.toString()).asDirectory();
  List childrenList=ReadOnlyList.Util.asList(dir1Node.getChildrenList(Snapshot.CURRENT_STATE_ID));
  assertEquals(1,childrenList.size());
  INode fooNode=childrenList.get(0);
  assertTrue(fooNode.asDirectory().isWithSnapshot());
  assertSame(dir1Node,fooNode.getParent());
  List diffList=dir1Node.getDiffs().asList();
  assertEquals(1,diffList.size());
  DirectoryDiff diff=diffList.get(0);
  assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
  assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
  INodeDirectory dir2Node=fsdir.getINode4Write(dir2.toString()).asDirectory();
  assertTrue(dir2Node.isSnapshottable());
  Quota.Counts counts=dir2Node.computeQuotaUsage();
  assertEquals(4,counts.get(Quota.NAMESPACE));
  assertEquals(0,counts.get(Quota.DISKSPACE));
  childrenList=ReadOnlyList.Util.asList(dir2Node.asDirectory().getChildrenList(Snapshot.CURRENT_STATE_ID));
  assertEquals(1,childrenList.size());
  INode subdir2Node=childrenList.get(0);
  assertTrue(subdir2Node.asDirectory().isWithSnapshot());
  assertSame(dir2Node,subdir2Node.getParent());
  assertSame(subdir2Node,fsdir.getINode4Write(sub_dir2.toString()));
  INode subsubdir2Node=fsdir.getINode4Write(subsub_dir2.toString());
  assertTrue(subsubdir2Node.getClass() == INodeDirectory.class);
  assertSame(subdir2Node,subsubdir2Node.getParent());
  diffList=(dir2Node).getDiffs().asList();
  assertEquals(1,diffList.size());
  diff=diffList.get(0);
  assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
  assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
  diffList=subdir2Node.asDirectory().getDiffs().asList();
  assertEquals(0,diffList.size());
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier EqualityVerifier HybridVerifier 

/** 
 * Test rename to an invalid name (xxx/.snapshot)
 */
@Test public void testRenameUndo_7() throws Exception {
  final Path root=new Path("/");
  final Path foo=new Path(root,"foo");
  final Path bar=new Path(foo,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,root,snap1);
  final Path invalid=new Path(foo,HdfsConstants.DOT_SNAPSHOT_DIR);
  try {
    hdfs.rename(bar,invalid);
    fail("expect exception since invalid name is used for rename");
  }
 catch (  Exception e) {
    GenericTestUtils.assertExceptionContains("\"" + HdfsConstants.DOT_SNAPSHOT_DIR + "\" is a reserved name",e);
  }
  INodeDirectory rootNode=fsdir.getINode4Write(root.toString()).asDirectory();
  INodeDirectory fooNode=fsdir.getINode4Write(foo.toString()).asDirectory();
  ReadOnlyList children=fooNode.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(1,children.size());
  List diffList=fooNode.getDiffs().asList();
  assertEquals(1,diffList.size());
  DirectoryDiff diff=diffList.get(0);
  Snapshot s1=rootNode.getSnapshot(DFSUtil.string2Bytes(snap1));
  assertEquals(s1.getId(),diff.getSnapshotId());
  assertTrue(diff.getChildrenDiff().getList(ListType.DELETED).isEmpty());
  assertTrue(diff.getChildrenDiff().getList(ListType.CREATED).isEmpty());
  INodeFile barNode=fsdir.getINode4Write(bar.toString()).asFile();
  assertSame(barNode,children.get(0));
  assertSame(fooNode,barNode.getParent());
  List barDiffList=barNode.getDiffs().asList();
  assertEquals(1,barDiffList.size());
  FileDiff barDiff=barDiffList.get(0);
  assertEquals(s1.getId(),barDiff.getSnapshotId());
  hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
  hdfs.saveNamespace();
  hdfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
  cluster.shutdown();
  cluster=new MiniDFSCluster.Builder(conf).format(false).numDataNodes(REPL).build();
  cluster.waitActive();
  restartClusterAndCheckImage(true);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Rename a single file across snapshottable dirs.
 */
@Test(timeout=60000) public void testRenameFileAcrossSnapshottableDirs() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  final Path foo=new Path(sdir2,"foo");
  DFSTestUtil.createFile(hdfs,foo,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2");
  hdfs.createSnapshot(sdir1,"s3");
  final Path newfoo=new Path(sdir1,"foo");
  hdfs.rename(foo,newfoo);
  hdfs.setReplication(newfoo,REPL_1);
  final Path foo_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo");
  assertTrue(hdfs.exists(foo_s2));
  FileStatus status=hdfs.getFileStatus(foo_s2);
  assertEquals(REPL,status.getReplication());
  final Path foo_s3=SnapshotTestHelper.getSnapshotPath(sdir1,"s3","foo");
  assertFalse(hdfs.exists(foo_s3));
  INodeDirectory sdir2Node=fsdir.getINode(sdir2.toString()).asDirectory();
  Snapshot s2=sdir2Node.getSnapshot(DFSUtil.string2Bytes("s2"));
  INodeFile sfoo=fsdir.getINode(newfoo.toString()).asFile();
  assertEquals(s2.getId(),sfoo.getDiffs().getLastSnapshotId());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * After the following steps:
 * 
 * 1. Take snapshot s1 on /dir1 at time t1.
 * 2. Take snapshot s2 on /dir2 at time t2.
 * 3. Modify the subtree of /dir2/foo/ to make it a dir with snapshots.
 * 4. Take snapshot s3 on /dir1 at time t3.
 * 5. Rename /dir2/foo/ to /dir1/foo/.
 * 
 * When changes happening on foo, the diff should be recorded in snapshot s2. 
 */
@Test(timeout=60000) public void testRenameDirAcrossSnapshottableDirs() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  final Path foo=new Path(sdir2,"foo");
  final Path bar=new Path(foo,"bar");
  final Path bar2=new Path(foo,"bar2");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPL,SEED);
  DFSTestUtil.createFile(hdfs,bar2,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2");
  hdfs.setReplication(bar2,REPL_1);
  hdfs.delete(bar,true);
  hdfs.createSnapshot(sdir1,"s3");
  final Path newfoo=new Path(sdir1,"foo");
  hdfs.rename(foo,newfoo);
  final Path snapshotBar=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo/bar");
  assertTrue(hdfs.exists(snapshotBar));
  final Path newBar2=new Path(newfoo,"bar2");
  assertTrue(hdfs.exists(newBar2));
  hdfs.delete(newBar2,true);
  final Path bar2_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo/bar2");
  assertTrue(hdfs.exists(bar2_s2));
  FileStatus status=hdfs.getFileStatus(bar2_s2);
  assertEquals(REPL,status.getReplication());
  final Path bar2_s3=SnapshotTestHelper.getSnapshotPath(sdir1,"s3","foo/bar2");
  assertFalse(hdfs.exists(bar2_s3));
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * After rename, delete the snapshot in src
 */
@Test public void testRenameDirAndDeleteSnapshot_2() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  final Path foo=new Path(sdir2,"foo");
  final Path bar=new Path(foo,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s3");
  final Path newfoo=new Path(sdir1,"foo");
  hdfs.rename(foo,newfoo);
  restartClusterAndCheckImage(true);
  final Path bar2=new Path(newfoo,"bar2");
  DFSTestUtil.createFile(hdfs,bar2,BLOCKSIZE,REPL,SEED);
  hdfs.createSnapshot(sdir1,"s4");
  hdfs.delete(newfoo,true);
  final Path bar2_s4=SnapshotTestHelper.getSnapshotPath(sdir1,"s4","foo/bar2");
  assertTrue(hdfs.exists(bar2_s4));
  final Path bar_s4=SnapshotTestHelper.getSnapshotPath(sdir1,"s4","foo/bar");
  assertTrue(hdfs.exists(bar_s4));
  hdfs.deleteSnapshot(sdir1,"s4");
  restartClusterAndCheckImage(true);
  Path bar_s3=SnapshotTestHelper.getSnapshotPath(sdir1,"s3","foo/bar");
  assertFalse(hdfs.exists(bar_s3));
  bar_s3=SnapshotTestHelper.getSnapshotPath(sdir2,"s3","foo/bar");
  assertTrue(hdfs.exists(bar_s3));
  Path bar2_s3=SnapshotTestHelper.getSnapshotPath(sdir1,"s3","foo/bar2");
  assertFalse(hdfs.exists(bar2_s3));
  bar2_s3=SnapshotTestHelper.getSnapshotPath(sdir2,"s3","foo/bar2");
  assertFalse(hdfs.exists(bar2_s3));
  hdfs.deleteSnapshot(sdir2,"s3");
  final Path bar_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo/bar");
  assertTrue(hdfs.exists(bar_s2));
  INodeDirectory sdir2Node=fsdir.getINode(sdir2.toString()).asDirectory();
  Snapshot s2=sdir2Node.getSnapshot(DFSUtil.string2Bytes("s2"));
  final Path foo_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo");
  INodeReference fooRef=fsdir.getINode(foo_s2.toString()).asReference();
  assertTrue(fooRef instanceof INodeReference.WithName);
  INodeReference.WithCount fooWC=(WithCount)fooRef.getReferredINode();
  assertEquals(1,fooWC.getReferenceCount());
  INodeDirectory fooDir=fooWC.getReferredINode().asDirectory();
  List diffs=fooDir.getDiffs().asList();
  assertEquals(1,diffs.size());
  assertEquals(s2.getId(),diffs.get(0).getSnapshotId());
  restartClusterAndCheckImage(true);
  hdfs.deleteSnapshot(sdir2,"s2");
  assertFalse(hdfs.exists(bar_s2));
  restartClusterAndCheckImage(true);
  Quota.Counts q=fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
  assertEquals(4,q.get(Quota.NAMESPACE));
  assertEquals(0,q.get(Quota.DISKSPACE));
  hdfs.deleteSnapshot(sdir1,"s1");
  restartClusterAndCheckImage(true);
  q=fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
  assertEquals(3,q.get(Quota.NAMESPACE));
  assertEquals(0,q.get(Quota.DISKSPACE));
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test renaming a file and then delete snapshots.
 */
@Test public void testRenameFileAndDeleteSnapshot() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  final Path foo=new Path(sdir2,"foo");
  DFSTestUtil.createFile(hdfs,foo,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2");
  hdfs.createSnapshot(sdir1,"s3");
  final Path newfoo=new Path(sdir1,"foo");
  hdfs.rename(foo,newfoo);
  hdfs.setReplication(newfoo,REPL_1);
  hdfs.createSnapshot(sdir1,"s4");
  hdfs.setReplication(newfoo,REPL_2);
  FileStatus status=hdfs.getFileStatus(newfoo);
  assertEquals(REPL_2,status.getReplication());
  final Path foo_s4=SnapshotTestHelper.getSnapshotPath(sdir1,"s4","foo");
  status=hdfs.getFileStatus(foo_s4);
  assertEquals(REPL_1,status.getReplication());
  hdfs.createSnapshot(sdir1,"s5");
  final Path foo_s5=SnapshotTestHelper.getSnapshotPath(sdir1,"s5","foo");
  status=hdfs.getFileStatus(foo_s5);
  assertEquals(REPL_2,status.getReplication());
  hdfs.deleteSnapshot(sdir1,"s5");
  restartClusterAndCheckImage(true);
  assertFalse(hdfs.exists(foo_s5));
  status=hdfs.getFileStatus(foo_s4);
  assertEquals(REPL_1,status.getReplication());
  hdfs.deleteSnapshot(sdir1,"s4");
  assertFalse(hdfs.exists(foo_s4));
  Path foo_s3=SnapshotTestHelper.getSnapshotPath(sdir1,"s3","foo");
  assertFalse(hdfs.exists(foo_s3));
  foo_s3=SnapshotTestHelper.getSnapshotPath(sdir2,"s3","foo");
  assertFalse(hdfs.exists(foo_s3));
  final Path foo_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo");
  assertTrue(hdfs.exists(foo_s2));
  status=hdfs.getFileStatus(foo_s2);
  assertEquals(REPL,status.getReplication());
  INodeFile snode=fsdir.getINode(newfoo.toString()).asFile();
  assertEquals(1,snode.getDiffs().asList().size());
  INodeDirectory sdir2Node=fsdir.getINode(sdir2.toString()).asDirectory();
  Snapshot s2=sdir2Node.getSnapshot(DFSUtil.string2Bytes("s2"));
  assertEquals(s2.getId(),snode.getDiffs().getLastSnapshotId());
  restartClusterAndCheckImage(true);
  hdfs.deleteSnapshot(sdir2,"s2");
  assertFalse(hdfs.exists(foo_s2));
  restartClusterAndCheckImage(true);
  hdfs.deleteSnapshot(sdir1,"s3");
  restartClusterAndCheckImage(true);
  hdfs.deleteSnapshot(sdir1,"s1");
  restartClusterAndCheckImage(true);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Test renaming a dir and then delete snapshots.
 */
@Test public void testRenameDirAndDeleteSnapshot_1() throws Exception {
  final Path sdir1=new Path("/dir1");
  final Path sdir2=new Path("/dir2");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  final Path foo=new Path(sdir2,"foo");
  final Path bar=new Path(foo,"bar");
  final Path bar2=new Path(foo,"bar2");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPL,SEED);
  DFSTestUtil.createFile(hdfs,bar2,BLOCKSIZE,REPL,SEED);
  SnapshotTestHelper.createSnapshot(hdfs,sdir1,"s1");
  SnapshotTestHelper.createSnapshot(hdfs,sdir2,"s2");
  hdfs.createSnapshot(sdir1,"s3");
  final Path newfoo=new Path(sdir1,"foo");
  hdfs.rename(foo,newfoo);
  final Path newbar=new Path(newfoo,bar.getName());
  final Path newbar2=new Path(newfoo,bar2.getName());
  final Path newbar3=new Path(newfoo,"bar3");
  DFSTestUtil.createFile(hdfs,newbar3,BLOCKSIZE,REPL,SEED);
  hdfs.createSnapshot(sdir1,"s4");
  hdfs.delete(newbar,true);
  hdfs.delete(newbar3,true);
  assertFalse(hdfs.exists(newbar3));
  assertFalse(hdfs.exists(bar));
  final Path bar_s4=SnapshotTestHelper.getSnapshotPath(sdir1,"s4","foo/bar");
  final Path bar3_s4=SnapshotTestHelper.getSnapshotPath(sdir1,"s4","foo/bar3");
  assertTrue(hdfs.exists(bar_s4));
  assertTrue(hdfs.exists(bar3_s4));
  hdfs.createSnapshot(sdir1,"s5");
  hdfs.delete(newbar2,true);
  assertFalse(hdfs.exists(bar2));
  final Path bar2_s5=SnapshotTestHelper.getSnapshotPath(sdir1,"s5","foo/bar2");
  assertTrue(hdfs.exists(bar2_s5));
  hdfs.deleteSnapshot(sdir1,"s5");
  restartClusterAndCheckImage(true);
  assertFalse(hdfs.exists(bar2_s5));
  final Path bar2_s4=SnapshotTestHelper.getSnapshotPath(sdir1,"s4","foo/bar2");
  assertTrue(hdfs.exists(bar2_s4));
  hdfs.deleteSnapshot(sdir1,"s4");
  assertFalse(hdfs.exists(bar_s4));
  Path bar_s3=SnapshotTestHelper.getSnapshotPath(sdir1,"s3","foo/bar");
  assertFalse(hdfs.exists(bar_s3));
  bar_s3=SnapshotTestHelper.getSnapshotPath(sdir2,"s3","foo/bar");
  assertFalse(hdfs.exists(bar_s3));
  final Path bar_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo/bar");
  assertTrue(hdfs.exists(bar_s2));
  assertFalse(hdfs.exists(bar2_s4));
  Path bar2_s3=SnapshotTestHelper.getSnapshotPath(sdir1,"s3","foo/bar2");
  assertFalse(hdfs.exists(bar2_s3));
  bar2_s3=SnapshotTestHelper.getSnapshotPath(sdir2,"s3","foo/bar2");
  assertFalse(hdfs.exists(bar2_s3));
  final Path bar2_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo/bar2");
  assertTrue(hdfs.exists(bar2_s2));
  assertFalse(hdfs.exists(bar3_s4));
  Path bar3_s3=SnapshotTestHelper.getSnapshotPath(sdir1,"s3","foo/bar3");
  assertFalse(hdfs.exists(bar3_s3));
  bar3_s3=SnapshotTestHelper.getSnapshotPath(sdir2,"s3","foo/bar3");
  assertFalse(hdfs.exists(bar3_s3));
  final Path bar3_s2=SnapshotTestHelper.getSnapshotPath(sdir2,"s2","foo/bar3");
  assertFalse(hdfs.exists(bar3_s2));
  restartClusterAndCheckImage(true);
  hdfs.deleteSnapshot(sdir2,"s2");
  assertFalse(hdfs.exists(bar_s2));
  assertFalse(hdfs.exists(bar2_s2));
  restartClusterAndCheckImage(true);
  hdfs.deleteSnapshot(sdir1,"s3");
  restartClusterAndCheckImage(true);
  hdfs.deleteSnapshot(sdir1,"s1");
  restartClusterAndCheckImage(true);
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestSetQuotaWithSnapshot

APIUtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier EqualityVerifier HybridVerifier 

/** 
 * Test clear quota of a snapshottable dir or a dir with snapshot.
 */
@Test public void testClearQuota() throws Exception {
  final Path dir=new Path("/TestSnapshot");
  hdfs.mkdirs(dir);
  hdfs.allowSnapshot(dir);
  hdfs.setQuota(dir,HdfsConstants.QUOTA_DONT_SET,HdfsConstants.QUOTA_DONT_SET);
  INodeDirectory dirNode=fsdir.getINode4Write(dir.toString()).asDirectory();
  assertTrue(dirNode.isSnapshottable());
  assertEquals(0,dirNode.getDiffs().asList().size());
  hdfs.setQuota(dir,HdfsConstants.QUOTA_DONT_SET - 1,HdfsConstants.QUOTA_DONT_SET - 1);
  dirNode=fsdir.getINode4Write(dir.toString()).asDirectory();
  assertTrue(dirNode.isSnapshottable());
  assertEquals(0,dirNode.getDiffs().asList().size());
  hdfs.setQuota(dir,HdfsConstants.QUOTA_RESET,HdfsConstants.QUOTA_RESET);
  dirNode=fsdir.getINode4Write(dir.toString()).asDirectory();
  assertTrue(dirNode.isSnapshottable());
  assertEquals(0,dirNode.getDiffs().asList().size());
  SnapshotTestHelper.createSnapshot(hdfs,dir,"s1");
  hdfs.setQuota(dir,HdfsConstants.QUOTA_RESET,HdfsConstants.QUOTA_RESET);
  dirNode=fsdir.getINode4Write(dir.toString()).asDirectory();
  assertTrue(dirNode.isSnapshottable());
  assertEquals(1,dirNode.getDiffs().asList().size());
  SnapshottableDirectoryStatus[] status=hdfs.getSnapshottableDirListing();
  assertEquals(1,status.length);
  assertEquals(dir,status[0].getFullPath());
  final Path subDir=new Path(dir,"sub");
  hdfs.mkdirs(subDir);
  hdfs.createSnapshot(dir,"s2");
  final Path file=new Path(subDir,"file");
  DFSTestUtil.createFile(hdfs,file,BLOCKSIZE,REPLICATION,seed);
  hdfs.setQuota(dir,HdfsConstants.QUOTA_RESET,HdfsConstants.QUOTA_RESET);
  INode subNode=fsdir.getINode4Write(subDir.toString());
  assertTrue(subNode.asDirectory().isWithSnapshot());
  List diffList=subNode.asDirectory().getDiffs().asList();
  assertEquals(1,diffList.size());
  Snapshot s2=dirNode.getSnapshot(DFSUtil.string2Bytes("s2"));
  assertEquals(s2.getId(),diffList.get(0).getSnapshotId());
  List createdList=diffList.get(0).getChildrenDiff().getList(ListType.CREATED);
  assertEquals(1,createdList.size());
  assertSame(fsdir.getINode4Write(file.toString()),createdList.get(0));
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=60000) public void testSetQuota() throws Exception {
  final Path dir=new Path("/TestSnapshot");
  hdfs.mkdirs(dir);
  SnapshotTestHelper.createSnapshot(hdfs,dir,"s1");
  Path sub=new Path(dir,"sub");
  hdfs.mkdirs(sub);
  Path fileInSub=new Path(sub,"file");
  DFSTestUtil.createFile(hdfs,fileInSub,BLOCKSIZE,REPLICATION,seed);
  INodeDirectory subNode=INodeDirectory.valueOf(fsdir.getINode(sub.toString()),sub);
  assertFalse(subNode.isWithSnapshot());
  hdfs.setQuota(sub,Long.MAX_VALUE - 1,Long.MAX_VALUE - 1);
  subNode=INodeDirectory.valueOf(fsdir.getINode(sub.toString()),sub);
  assertTrue(subNode.isQuotaSet());
  assertFalse(subNode.isWithSnapshot());
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshot

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * A simple test that updates a sub-directory of a snapshottable directory
 * with snapshots
 */
@Test(timeout=60000) public void testUpdateDirectory() throws Exception {
  Path dir=new Path("/dir");
  Path sub=new Path(dir,"sub");
  Path subFile=new Path(sub,"file");
  DFSTestUtil.createFile(hdfs,subFile,BLOCKSIZE,REPLICATION,seed);
  FileStatus oldStatus=hdfs.getFileStatus(sub);
  hdfs.allowSnapshot(dir);
  hdfs.createSnapshot(dir,"s1");
  hdfs.setTimes(sub,100L,100L);
  Path snapshotPath=SnapshotTestHelper.getSnapshotPath(dir,"s1","sub");
  FileStatus snapshotStatus=hdfs.getFileStatus(snapshotPath);
  assertEquals(oldStatus.getModificationTime(),snapshotStatus.getModificationTime());
  assertEquals(oldStatus.getAccessTime(),snapshotStatus.getAccessTime());
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshotBlocksMap

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Make sure we delete 0-sized block when deleting an INodeFileUCWithSnapshot
 */
@Test public void testDeletionWithZeroSizeBlock() throws Exception {
  final Path foo=new Path("/foo");
  final Path bar=new Path(foo,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPLICATION,0L);
  SnapshotTestHelper.createSnapshot(hdfs,foo,"s0");
  hdfs.append(bar);
  INodeFile barNode=fsdir.getINode4Write(bar.toString()).asFile();
  BlockInfo[] blks=barNode.getBlocks();
  assertEquals(1,blks.length);
  assertEquals(BLOCKSIZE,blks[0].getNumBytes());
  ExtendedBlock previous=new ExtendedBlock(fsn.getBlockPoolId(),blks[0]);
  cluster.getNameNodeRpc().addBlock(bar.toString(),hdfs.getClient().getClientName(),previous,null,barNode.getId(),null);
  SnapshotTestHelper.createSnapshot(hdfs,foo,"s1");
  barNode=fsdir.getINode4Write(bar.toString()).asFile();
  blks=barNode.getBlocks();
  assertEquals(2,blks.length);
  assertEquals(BLOCKSIZE,blks[0].getNumBytes());
  assertEquals(0,blks[1].getNumBytes());
  hdfs.delete(bar,true);
  final Path sbar=SnapshotTestHelper.getSnapshotPath(foo,"s1",bar.getName());
  barNode=fsdir.getINode(sbar.toString()).asFile();
  blks=barNode.getBlocks();
  assertEquals(1,blks.length);
  assertEquals(BLOCKSIZE,blks[0].getNumBytes());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * 1. rename under-construction file with 0-sized blocks after snapshot.
 * 2. delete the renamed directory.
 * make sure we delete the 0-sized block.
 * see HDFS-5476.
 */
@Test public void testDeletionWithZeroSizeBlock3() throws Exception {
  final Path foo=new Path("/foo");
  final Path subDir=new Path(foo,"sub");
  final Path bar=new Path(subDir,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPLICATION,0L);
  hdfs.append(bar);
  INodeFile barNode=fsdir.getINode4Write(bar.toString()).asFile();
  BlockInfo[] blks=barNode.getBlocks();
  assertEquals(1,blks.length);
  ExtendedBlock previous=new ExtendedBlock(fsn.getBlockPoolId(),blks[0]);
  cluster.getNameNodeRpc().addBlock(bar.toString(),hdfs.getClient().getClientName(),previous,null,barNode.getId(),null);
  SnapshotTestHelper.createSnapshot(hdfs,foo,"s1");
  final Path bar2=new Path(subDir,"bar2");
  hdfs.rename(bar,bar2);
  INodeFile bar2Node=fsdir.getINode4Write(bar2.toString()).asFile();
  blks=bar2Node.getBlocks();
  assertEquals(2,blks.length);
  assertEquals(BLOCKSIZE,blks[0].getNumBytes());
  assertEquals(0,blks[1].getNumBytes());
  hdfs.delete(subDir,true);
  final Path sbar=SnapshotTestHelper.getSnapshotPath(foo,"s1","sub/bar");
  barNode=fsdir.getINode(sbar.toString()).asFile();
  blks=barNode.getBlocks();
  assertEquals(1,blks.length);
  assertEquals(BLOCKSIZE,blks[0].getNumBytes());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Make sure we delete 0-sized block when deleting an under-construction file
 */
@Test public void testDeletionWithZeroSizeBlock2() throws Exception {
  final Path foo=new Path("/foo");
  final Path subDir=new Path(foo,"sub");
  final Path bar=new Path(subDir,"bar");
  DFSTestUtil.createFile(hdfs,bar,BLOCKSIZE,REPLICATION,0L);
  hdfs.append(bar);
  INodeFile barNode=fsdir.getINode4Write(bar.toString()).asFile();
  BlockInfo[] blks=barNode.getBlocks();
  assertEquals(1,blks.length);
  ExtendedBlock previous=new ExtendedBlock(fsn.getBlockPoolId(),blks[0]);
  cluster.getNameNodeRpc().addBlock(bar.toString(),hdfs.getClient().getClientName(),previous,null,barNode.getId(),null);
  SnapshotTestHelper.createSnapshot(hdfs,foo,"s1");
  barNode=fsdir.getINode4Write(bar.toString()).asFile();
  blks=barNode.getBlocks();
  assertEquals(2,blks.length);
  assertEquals(BLOCKSIZE,blks[0].getNumBytes());
  assertEquals(0,blks[1].getNumBytes());
  hdfs.delete(subDir,true);
  final Path sbar=SnapshotTestHelper.getSnapshotPath(foo,"s1","sub/bar");
  barNode=fsdir.getINode(sbar.toString()).asFile();
  blks=barNode.getBlocks();
  assertEquals(1,blks.length);
  assertEquals(BLOCKSIZE,blks[0].getNumBytes());
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier NullVerifier HybridVerifier 

/** 
 * Test deleting a file with snapshots. Need to check the blocksMap to make
 * sure the corresponding record is updated correctly.
 */
@Test(timeout=60000) public void testDeletionWithSnapshots() throws Exception {
  Path file0=new Path(sub1,"file0");
  Path file1=new Path(sub1,"file1");
  Path sub2=new Path(sub1,"sub2");
  Path file2=new Path(sub2,"file2");
  Path file3=new Path(sub1,"file3");
  Path file4=new Path(sub1,"file4");
  Path file5=new Path(sub1,"file5");
  DFSTestUtil.createFile(hdfs,file0,4 * BLOCKSIZE,REPLICATION,seed);
  DFSTestUtil.createFile(hdfs,file1,2 * BLOCKSIZE,REPLICATION,seed);
  DFSTestUtil.createFile(hdfs,file2,3 * BLOCKSIZE,REPLICATION,seed);
{
    final INodeFile f2=assertBlockCollection(file2.toString(),3,fsdir,blockmanager);
    BlockInfo[] blocks=f2.getBlocks();
    hdfs.delete(sub2,true);
    for (    BlockInfo b : blocks) {
      assertNull(blockmanager.getBlockCollection(b));
    }
  }
  final String[] snapshots={"s0","s1","s2"};
  DFSTestUtil.createFile(hdfs,file3,5 * BLOCKSIZE,REPLICATION,seed);
  SnapshotTestHelper.createSnapshot(hdfs,sub1,snapshots[0]);
  DFSTestUtil.createFile(hdfs,file4,1 * BLOCKSIZE,REPLICATION,seed);
  SnapshotTestHelper.createSnapshot(hdfs,sub1,snapshots[1]);
  DFSTestUtil.createFile(hdfs,file5,7 * BLOCKSIZE,REPLICATION,seed);
  SnapshotTestHelper.createSnapshot(hdfs,sub1,snapshots[2]);
{
    INodeFile f1=assertBlockCollection(file1.toString(),2,fsdir,blockmanager);
    Assert.assertSame(INodeFile.class,f1.getClass());
    hdfs.setReplication(file1,(short)2);
    f1=assertBlockCollection(file1.toString(),2,fsdir,blockmanager);
    assertTrue(f1.isWithSnapshot());
    assertFalse(f1.isUnderConstruction());
  }
  final INodeFile f0=assertBlockCollection(file0.toString(),4,fsdir,blockmanager);
  BlockInfo[] blocks0=f0.getBlocks();
  Path snapshotFile0=SnapshotTestHelper.getSnapshotPath(sub1,"s0",file0.getName());
  assertBlockCollection(snapshotFile0.toString(),4,fsdir,blockmanager);
  hdfs.delete(file0,true);
  for (  BlockInfo b : blocks0) {
    assertNotNull(blockmanager.getBlockCollection(b));
  }
  assertBlockCollection(snapshotFile0.toString(),4,fsdir,blockmanager);
  String s1f0=SnapshotTestHelper.getSnapshotPath(sub1,"s1",file0.getName()).toString();
  assertBlockCollection(s1f0,4,fsdir,blockmanager);
  hdfs.deleteSnapshot(sub1,"s1");
  for (  BlockInfo b : blocks0) {
    assertNotNull(blockmanager.getBlockCollection(b));
  }
  assertBlockCollection(snapshotFile0.toString(),4,fsdir,blockmanager);
  try {
    INodeFile.valueOf(fsdir.getINode(s1f0),s1f0);
    fail("Expect FileNotFoundException when identifying the INode in a deleted Snapshot");
  }
 catch (  IOException e) {
    assertExceptionContains("File does not exist: " + s1f0,e);
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshotDeletion

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * When combine two snapshots, make sure files/directories created after the 
 * prior snapshot get destroyed.
 */
@Test(timeout=300000) public void testCombineSnapshotDiff3() throws Exception {
  Path dir=new Path("/dir");
  Path subDir1=new Path(dir,"subdir1");
  Path subDir2=new Path(dir,"subdir2");
  hdfs.mkdirs(subDir2);
  Path subsubDir=new Path(subDir1,"subsubdir");
  hdfs.mkdirs(subsubDir);
  SnapshotTestHelper.createSnapshot(hdfs,dir,"s1");
  Path newDir=new Path(subsubDir,"newdir");
  Path newFile=new Path(newDir,"newfile");
  DFSTestUtil.createFile(hdfs,newFile,BLOCKSIZE,REPLICATION,seed);
  Path newFile2=new Path(subDir2,"newfile");
  DFSTestUtil.createFile(hdfs,newFile2,BLOCKSIZE,REPLICATION,seed);
  SnapshotTestHelper.createSnapshot(hdfs,dir,"s2");
  checkQuotaUsageComputation(dir,11,BLOCKSIZE * 2 * REPLICATION);
  hdfs.delete(subsubDir,true);
  hdfs.delete(subDir2,true);
  checkQuotaUsageComputation(dir,14,BLOCKSIZE * 2 * REPLICATION);
  hdfs.deleteSnapshot(dir,"s2");
  checkQuotaUsageComputation(dir,8,0);
  Path subdir1_s1=SnapshotTestHelper.getSnapshotPath(dir,"s1",subDir1.getName());
  Path subdir1_s2=SnapshotTestHelper.getSnapshotPath(dir,"s2",subDir1.getName());
  assertTrue(hdfs.exists(subdir1_s1));
  assertFalse(hdfs.exists(subdir1_s2));
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

/** 
 * Test deleting the earliest (first) snapshot. In this simplest scenario, the 
 * snapshots are taken on the same directory, and we do not need to combine
 * snapshot diffs.
 */
@Test(timeout=300000) public void testDeleteEarliestSnapshot1() throws Exception {
  Path file0=new Path(sub,"file0");
  Path file1=new Path(sub,"file1");
  DFSTestUtil.createFile(hdfs,file0,BLOCKSIZE,REPLICATION,seed);
  DFSTestUtil.createFile(hdfs,file1,BLOCKSIZE,REPLICATION,seed);
  String snapshotName="s1";
  try {
    hdfs.deleteSnapshot(sub,snapshotName);
    fail("SnapshotException expected: " + sub.toString() + " is not snapshottable yet");
  }
 catch (  Exception e) {
    GenericTestUtils.assertExceptionContains("Directory is not a snapshottable directory: " + sub,e);
  }
  hdfs.allowSnapshot(sub);
  try {
    hdfs.deleteSnapshot(sub,snapshotName);
    fail("SnapshotException expected: snapshot " + snapshotName + " does not exist for "+ sub.toString());
  }
 catch (  Exception e) {
    GenericTestUtils.assertExceptionContains("Cannot delete snapshot " + snapshotName + " from path "+ sub.toString()+ ": the snapshot does not exist.",e);
  }
  SnapshotTestHelper.createSnapshot(hdfs,sub,snapshotName);
  checkQuotaUsageComputation(sub,4,BLOCKSIZE * REPLICATION * 2);
  hdfs.deleteSnapshot(sub,snapshotName);
  checkQuotaUsageComputation(sub,3,BLOCKSIZE * REPLICATION * 2);
  hdfs.createSnapshot(sub,snapshotName);
  checkQuotaUsageComputation(sub,4,BLOCKSIZE * REPLICATION * 2);
  Path newFile=new Path(sub,"newFile");
  DFSTestUtil.createFile(hdfs,newFile,BLOCKSIZE,REPLICATION,seed);
  String snapshotName2="s2";
  hdfs.createSnapshot(sub,snapshotName2);
  checkQuotaUsageComputation(sub,6,BLOCKSIZE * REPLICATION * 3);
  Path ss=SnapshotTestHelper.getSnapshotPath(sub,snapshotName2,"newFile");
  FileStatus statusBeforeDeletion=hdfs.getFileStatus(ss);
  hdfs.deleteSnapshot(sub,snapshotName);
  checkQuotaUsageComputation(sub,5,BLOCKSIZE * REPLICATION * 3);
  FileStatus statusAfterDeletion=hdfs.getFileStatus(ss);
  System.out.println("Before deletion: " + statusBeforeDeletion.toString() + "\n"+ "After deletion: "+ statusAfterDeletion.toString());
  assertEquals(statusBeforeDeletion.toString(),statusAfterDeletion.toString());
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test deleting a directory which is a descendant of a snapshottable
 * directory. In the test we need to cover the following cases:
 * 
 * 1. Delete current INodeFile/INodeDirectory without taking any snapshot.
 * 2. Delete current INodeFile/INodeDirectory while snapshots have been taken 
 * on ancestor(s).
 * 3. Delete current INodeFileWithSnapshot.
 * 4. Delete current INodeDirectoryWithSnapshot.
 * 
 */
@Test(timeout=300000) public void testDeleteCurrentFileDirectory() throws Exception {
  Path deleteDir=new Path(subsub,"deleteDir");
  Path deleteFile=new Path(deleteDir,"deleteFile");
  Path noChangeDirParent=new Path(sub,"noChangeDirParent");
  Path noChangeDir=new Path(noChangeDirParent,"noChangeDir");
  Path noChangeFile=new Path(noChangeDir,"noChangeFile");
  DFSTestUtil.createFile(hdfs,deleteFile,BLOCKSIZE,REPLICATION,seed);
  DFSTestUtil.createFile(hdfs,noChangeFile,BLOCKSIZE,REPLICATION,seed);
  Path metaChangeFile1=new Path(subsub,"metaChangeFile1");
  DFSTestUtil.createFile(hdfs,metaChangeFile1,BLOCKSIZE,REPLICATION,seed);
  Path metaChangeFile2=new Path(noChangeDir,"metaChangeFile2");
  DFSTestUtil.createFile(hdfs,metaChangeFile2,BLOCKSIZE,REPLICATION,seed);
  hdfs.delete(deleteDir,true);
  SnapshotTestHelper.createSnapshot(hdfs,dir,"s0");
  Path tempDir=new Path(dir,"tempdir");
  Path tempFile=new Path(tempDir,"tempfile");
  DFSTestUtil.createFile(hdfs,tempFile,BLOCKSIZE,REPLICATION,seed);
  final INodeFile temp=TestSnapshotBlocksMap.assertBlockCollection(tempFile.toString(),1,fsdir,blockmanager);
  BlockInfo[] blocks=temp.getBlocks();
  hdfs.delete(tempDir,true);
  checkQuotaUsageComputation(dir,9L,BLOCKSIZE * REPLICATION * 3);
  for (  BlockInfo b : blocks) {
    assertNull(blockmanager.getBlockCollection(b));
  }
  Path newFileAfterS0=new Path(subsub,"newFile");
  DFSTestUtil.createFile(hdfs,newFileAfterS0,BLOCKSIZE,REPLICATION,seed);
  hdfs.setReplication(metaChangeFile1,REPLICATION_1);
  hdfs.setReplication(metaChangeFile2,REPLICATION_1);
  SnapshotTestHelper.createSnapshot(hdfs,dir,"s1");
  checkQuotaUsageComputation(dir,14L,BLOCKSIZE * REPLICATION * 4);
  Snapshot snapshot0=fsdir.getINode(dir.toString()).asDirectory().getSnapshot(DFSUtil.string2Bytes("s0"));
  Snapshot snapshot1=fsdir.getINode(dir.toString()).asDirectory().getSnapshot(DFSUtil.string2Bytes("s1"));
  hdfs.delete(noChangeDirParent,true);
  checkQuotaUsageComputation(dir,17L,BLOCKSIZE * REPLICATION * 4);
  Path snapshotNoChangeDir=SnapshotTestHelper.getSnapshotPath(dir,"s1",sub.getName() + "/" + noChangeDirParent.getName()+ "/"+ noChangeDir.getName());
  INodeDirectory snapshotNode=(INodeDirectory)fsdir.getINode(snapshotNoChangeDir.toString());
  assertEquals(INodeDirectory.class,snapshotNode.getClass());
  ReadOnlyList children=snapshotNode.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(2,children.size());
  INode noChangeFileSCopy=children.get(1);
  assertEquals(noChangeFile.getName(),noChangeFileSCopy.getLocalName());
  assertEquals(INodeFile.class,noChangeFileSCopy.getClass());
  TestSnapshotBlocksMap.assertBlockCollection(new Path(snapshotNoChangeDir,noChangeFileSCopy.getLocalName()).toString(),1,fsdir,blockmanager);
  INodeFile metaChangeFile2SCopy=children.get(0).asFile();
  assertEquals(metaChangeFile2.getName(),metaChangeFile2SCopy.getLocalName());
  assertTrue(metaChangeFile2SCopy.isWithSnapshot());
  assertFalse(metaChangeFile2SCopy.isUnderConstruction());
  TestSnapshotBlocksMap.assertBlockCollection(new Path(snapshotNoChangeDir,metaChangeFile2SCopy.getLocalName()).toString(),1,fsdir,blockmanager);
  assertEquals(REPLICATION_1,metaChangeFile2SCopy.getFileReplication(Snapshot.CURRENT_STATE_ID));
  assertEquals(REPLICATION_1,metaChangeFile2SCopy.getFileReplication(snapshot1.getId()));
  assertEquals(REPLICATION,metaChangeFile2SCopy.getFileReplication(snapshot0.getId()));
  Path newFile=new Path(sub,"newFile");
  DFSTestUtil.createFile(hdfs,newFile,BLOCKSIZE,REPLICATION,seed);
  final INodeFile newFileNode=TestSnapshotBlocksMap.assertBlockCollection(newFile.toString(),1,fsdir,blockmanager);
  blocks=newFileNode.getBlocks();
  checkQuotaUsageComputation(dir,18L,BLOCKSIZE * REPLICATION * 5);
  hdfs.delete(sub,true);
  checkQuotaUsageComputation(dir,19L,BLOCKSIZE * REPLICATION * 4);
  for (  BlockInfo b : blocks) {
    assertNull(blockmanager.getBlockCollection(b));
  }
  Path snapshotSub=SnapshotTestHelper.getSnapshotPath(dir,"s1",sub.getName());
  INodeDirectory snapshotNode4Sub=fsdir.getINode(snapshotSub.toString()).asDirectory();
  assertTrue(snapshotNode4Sub.isWithSnapshot());
  assertEquals(1,snapshotNode4Sub.getChildrenList(Snapshot.CURRENT_STATE_ID).size());
  assertEquals(2,snapshotNode4Sub.getChildrenList(snapshot1.getId()).size());
  INode snapshotNode4Subsub=snapshotNode4Sub.getChildrenList(Snapshot.CURRENT_STATE_ID).get(0);
  assertTrue(snapshotNode4Subsub.asDirectory().isWithSnapshot());
  assertTrue(snapshotNode4Sub == snapshotNode4Subsub.getParent());
  INodeDirectory snapshotSubsubDir=(INodeDirectory)snapshotNode4Subsub;
  children=snapshotSubsubDir.getChildrenList(Snapshot.CURRENT_STATE_ID);
  assertEquals(2,children.size());
  assertEquals(children.get(0).getLocalName(),metaChangeFile1.getName());
  assertEquals(children.get(1).getLocalName(),newFileAfterS0.getName());
  children=snapshotSubsubDir.getChildrenList(snapshot0.getId());
  assertEquals(1,children.size());
  INode child=children.get(0);
  assertEquals(child.getLocalName(),metaChangeFile1.getName());
  INodeFile metaChangeFile1SCopy=child.asFile();
  assertTrue(metaChangeFile1SCopy.isWithSnapshot());
  assertFalse(metaChangeFile1SCopy.isUnderConstruction());
  assertEquals(REPLICATION_1,metaChangeFile1SCopy.getFileReplication(Snapshot.CURRENT_STATE_ID));
  assertEquals(REPLICATION_1,metaChangeFile1SCopy.getFileReplication(snapshot1.getId()));
  assertEquals(REPLICATION,metaChangeFile1SCopy.getFileReplication(snapshot0.getId()));
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier EqualityVerifier HybridVerifier 

/** 
 * A test covering the case where the snapshot diff to be deleted is renamed 
 * to its previous snapshot. 
 */
@Test(timeout=300000) public void testRenameSnapshotDiff() throws Exception {
  cluster.getNamesystem().getSnapshotManager().setAllowNestedSnapshots(true);
  final Path subFile0=new Path(sub,"file0");
  final Path subsubFile0=new Path(subsub,"file0");
  DFSTestUtil.createFile(hdfs,subFile0,BLOCKSIZE,REPLICATION,seed);
  DFSTestUtil.createFile(hdfs,subsubFile0,BLOCKSIZE,REPLICATION,seed);
  hdfs.setOwner(subsub,"owner","group");
  SnapshotTestHelper.createSnapshot(hdfs,sub,"s0");
  checkQuotaUsageComputation(sub,5,BLOCKSIZE * 6);
  final Path subFile1=new Path(sub,"file1");
  final Path subsubFile1=new Path(subsub,"file1");
  DFSTestUtil.createFile(hdfs,subFile1,BLOCKSIZE,REPLICATION_1,seed);
  DFSTestUtil.createFile(hdfs,subsubFile1,BLOCKSIZE,REPLICATION,seed);
  checkQuotaUsageComputation(sub,8,BLOCKSIZE * 11);
  SnapshotTestHelper.createSnapshot(hdfs,sub,"s1");
  checkQuotaUsageComputation(sub,9,BLOCKSIZE * 11);
  SnapshotTestHelper.createSnapshot(hdfs,dir,"s2");
  checkQuotaUsageComputation(dir,11,BLOCKSIZE * 11);
  checkQuotaUsageComputation(sub,9,BLOCKSIZE * 11);
  hdfs.setOwner(subsub,"unknown","unknown");
  hdfs.setReplication(subsubFile1,REPLICATION_1);
  checkQuotaUsageComputation(dir,13,BLOCKSIZE * 11);
  checkQuotaUsageComputation(sub,11,BLOCKSIZE * 11);
  hdfs.delete(subFile1,true);
  checkQuotaUsageComputation(new Path("/"),16,BLOCKSIZE * 11);
  checkQuotaUsageComputation(dir,15,BLOCKSIZE * 11);
  checkQuotaUsageComputation(sub,13,BLOCKSIZE * 11);
  Path subsubSnapshotCopy=SnapshotTestHelper.getSnapshotPath(dir,"s2",sub.getName() + Path.SEPARATOR + subsub.getName());
  Path subsubFile1SCopy=SnapshotTestHelper.getSnapshotPath(dir,"s2",sub.getName() + Path.SEPARATOR + subsub.getName()+ Path.SEPARATOR+ subsubFile1.getName());
  Path subFile1SCopy=SnapshotTestHelper.getSnapshotPath(dir,"s2",sub.getName() + Path.SEPARATOR + subFile1.getName());
  FileStatus subsubStatus=hdfs.getFileStatus(subsubSnapshotCopy);
  assertEquals("owner",subsubStatus.getOwner());
  assertEquals("group",subsubStatus.getGroup());
  FileStatus subsubFile1Status=hdfs.getFileStatus(subsubFile1SCopy);
  assertEquals(REPLICATION,subsubFile1Status.getReplication());
  FileStatus subFile1Status=hdfs.getFileStatus(subFile1SCopy);
  assertEquals(REPLICATION_1,subFile1Status.getReplication());
  hdfs.deleteSnapshot(dir,"s2");
  checkQuotaUsageComputation(new Path("/"),14,BLOCKSIZE * 11);
  checkQuotaUsageComputation(dir,13,BLOCKSIZE * 11);
  checkQuotaUsageComputation(sub,12,BLOCKSIZE * 11);
  try {
    hdfs.getFileStatus(subsubSnapshotCopy);
    fail("should throw FileNotFoundException");
  }
 catch (  FileNotFoundException e) {
    GenericTestUtils.assertExceptionContains("File does not exist: " + subsubSnapshotCopy.toString(),e);
  }
  try {
    hdfs.getFileStatus(subsubFile1SCopy);
    fail("should throw FileNotFoundException");
  }
 catch (  FileNotFoundException e) {
    GenericTestUtils.assertExceptionContains("File does not exist: " + subsubFile1SCopy.toString(),e);
  }
  try {
    hdfs.getFileStatus(subFile1SCopy);
    fail("should throw FileNotFoundException");
  }
 catch (  FileNotFoundException e) {
    GenericTestUtils.assertExceptionContains("File does not exist: " + subFile1SCopy.toString(),e);
  }
  subsubSnapshotCopy=SnapshotTestHelper.getSnapshotPath(sub,"s1",subsub.getName());
  subsubFile1SCopy=SnapshotTestHelper.getSnapshotPath(sub,"s1",subsub.getName() + Path.SEPARATOR + subsubFile1.getName());
  subFile1SCopy=SnapshotTestHelper.getSnapshotPath(sub,"s1",subFile1.getName());
  subsubStatus=hdfs.getFileStatus(subsubSnapshotCopy);
  assertEquals("owner",subsubStatus.getOwner());
  assertEquals("group",subsubStatus.getGroup());
  subsubFile1Status=hdfs.getFileStatus(subsubFile1SCopy);
  assertEquals(REPLICATION,subsubFile1Status.getReplication());
  subFile1Status=hdfs.getFileStatus(subFile1SCopy);
  assertEquals(REPLICATION_1,subFile1Status.getReplication());
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshotFileLength

APIUtilityVerifier InternalCallVerifier ConditionMatcher PublicFieldVerifier 

/** 
 * Test that we cannot read a file beyond its snapshot length
 * when accessing it via a snapshot path.
 */
@Test(timeout=300000) public void testSnapshotfileLength() throws Exception {
  hdfs.mkdirs(sub);
  int bytesRead;
  byte[] buffer=new byte[BLOCKSIZE * 8];
  FSDataInputStream fis=null;
  FileStatus fileStatus=null;
  Path file1=new Path(sub,file1Name);
  DFSTestUtil.createFile(hdfs,file1,0,REPLICATION,SEED);
  DFSTestUtil.appendFile(hdfs,file1,BLOCKSIZE);
  hdfs.allowSnapshot(sub);
  hdfs.createSnapshot(sub,snapshot1);
  DFSTestUtil.appendFile(hdfs,file1,BLOCKSIZE);
  fileStatus=hdfs.getFileStatus(file1);
  assertThat(fileStatus.getLen(),is((long)BLOCKSIZE * 2));
  fis=hdfs.open(file1);
  bytesRead=fis.read(0,buffer,0,buffer.length);
  assertThat(bytesRead,is(BLOCKSIZE * 2));
  fis.close();
  Path file1snap1=SnapshotTestHelper.getSnapshotPath(sub,snapshot1,file1Name);
  fis=hdfs.open(file1snap1);
  fileStatus=hdfs.getFileStatus(file1snap1);
  assertThat(fileStatus.getLen(),is((long)BLOCKSIZE));
  bytesRead=fis.read(0,buffer,0,buffer.length);
  assertThat(bytesRead,is(BLOCKSIZE));
  fis.close();
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier 

/** 
 * Adding as part of jira HDFS-5343
 * Test for checking the cat command on snapshot path it
 * cannot read a file beyond snapshot file length
 * @throws Exception
 */
@Test(timeout=600000) public void testSnapshotFileLengthWithCatCommand() throws Exception {
  FSDataInputStream fis=null;
  FileStatus fileStatus=null;
  int bytesRead;
  byte[] buffer=new byte[BLOCKSIZE * 8];
  hdfs.mkdirs(sub);
  Path file1=new Path(sub,file1Name);
  DFSTestUtil.createFile(hdfs,file1,BLOCKSIZE,REPLICATION,SEED);
  hdfs.allowSnapshot(sub);
  hdfs.createSnapshot(sub,snapshot1);
  DFSTestUtil.appendFile(hdfs,file1,BLOCKSIZE);
  fileStatus=hdfs.getFileStatus(file1);
  assertEquals("Unexpected file length",BLOCKSIZE * 2,fileStatus.getLen());
  fis=hdfs.open(file1);
  bytesRead=fis.read(buffer,0,buffer.length);
  assertEquals("Unexpected # bytes read",BLOCKSIZE * 2,bytesRead);
  fis.close();
  Path file1snap1=SnapshotTestHelper.getSnapshotPath(sub,snapshot1,file1Name);
  fis=hdfs.open(file1snap1);
  fileStatus=hdfs.getFileStatus(file1snap1);
  assertEquals(fileStatus.getLen(),BLOCKSIZE);
  bytesRead=fis.read(buffer,0,buffer.length);
  assertEquals("Unexpected # bytes read",BLOCKSIZE,bytesRead);
  fis.close();
  PrintStream outBackup=System.out;
  PrintStream errBackup=System.err;
  ByteArrayOutputStream bao=new ByteArrayOutputStream();
  System.setOut(new PrintStream(bao));
  System.setErr(new PrintStream(bao));
  FsShell shell=new FsShell();
  try {
    ToolRunner.run(conf,shell,new String[]{"-cat","/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1"});
    assertEquals("Unexpected # bytes from -cat",BLOCKSIZE,bao.size());
  }
  finally {
    System.setOut(outBackup);
    System.setErr(errBackup);
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshotRename

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test FileStatus of snapshot file before/after rename
 */
@Test(timeout=60000) public void testSnapshotRename() throws Exception {
  DFSTestUtil.createFile(hdfs,file1,BLOCKSIZE,REPLICATION,seed);
  Path snapshotRoot=SnapshotTestHelper.createSnapshot(hdfs,sub1,"s1");
  Path ssPath=new Path(snapshotRoot,file1.getName());
  assertTrue(hdfs.exists(ssPath));
  FileStatus statusBeforeRename=hdfs.getFileStatus(ssPath);
  hdfs.renameSnapshot(sub1,"s1","s2");
  assertFalse(hdfs.exists(ssPath));
  snapshotRoot=SnapshotTestHelper.getSnapshotRoot(sub1,"s2");
  ssPath=new Path(snapshotRoot,file1.getName());
  assertTrue(hdfs.exists(ssPath));
  FileStatus statusAfterRename=hdfs.getFileStatus(ssPath);
  assertFalse(statusBeforeRename.equals(statusAfterRename));
  statusBeforeRename.setPath(statusAfterRename.getPath());
  assertEquals(statusBeforeRename.toString(),statusAfterRename.toString());
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshotReplication

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test replication number calculation for a file with snapshots.
 */
@Test(timeout=60000) public void testReplicationWithSnapshot() throws Exception {
  short fileRep=1;
  DFSTestUtil.createFile(hdfs,file1,BLOCKSIZE,fileRep,seed);
  Map snapshotRepMap=new HashMap();
  for (; fileRep < NUMDATANODE; ) {
    Path snapshotRoot=SnapshotTestHelper.createSnapshot(hdfs,sub1,"s" + fileRep);
    Path snapshot=new Path(snapshotRoot,file1.getName());
    assertEquals(fileRep,getINodeFile(snapshot).getFileReplication());
    snapshotRepMap.put(snapshot,fileRep);
    hdfs.setReplication(file1,++fileRep);
    checkFileReplication(file1,fileRep,fileRep);
    checkSnapshotFileReplication(file1,snapshotRepMap,fileRep);
  }
  hdfs.setReplication(file1,REPLICATION);
  checkFileReplication(file1,REPLICATION,(short)(NUMDATANODE - 1));
  checkSnapshotFileReplication(file1,snapshotRepMap,(short)(NUMDATANODE - 1));
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test replication for a file with snapshots, also including the scenario
 * where the original file is deleted
 */
@Test(timeout=60000) public void testReplicationAfterDeletion() throws Exception {
  DFSTestUtil.createFile(hdfs,file1,BLOCKSIZE,REPLICATION,seed);
  Map snapshotRepMap=new HashMap();
  for (int i=1; i <= 3; i++) {
    Path root=SnapshotTestHelper.createSnapshot(hdfs,sub1,"s" + i);
    Path ssFile=new Path(root,file1.getName());
    snapshotRepMap.put(ssFile,REPLICATION);
  }
  checkFileReplication(file1,REPLICATION,REPLICATION);
  checkSnapshotFileReplication(file1,snapshotRepMap,REPLICATION);
  hdfs.delete(file1,true);
  for (  Path ss : snapshotRepMap.keySet()) {
    final INodeFile ssInode=getINodeFile(ss);
    assertEquals(REPLICATION,ssInode.getBlockReplication());
    assertEquals(snapshotRepMap.get(ss).shortValue(),ssInode.getFileReplication());
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshotStatsMXBean

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test getting SnapshotStatsMXBean information
 */
@Test public void testSnapshotStatsMXBeanInfo() throws Exception {
  Configuration conf=new Configuration();
  MiniDFSCluster cluster=null;
  String pathName="/snapshot";
  Path path=new Path(pathName);
  try {
    cluster=new MiniDFSCluster.Builder(conf).build();
    cluster.waitActive();
    SnapshotManager sm=cluster.getNamesystem().getSnapshotManager();
    DistributedFileSystem dfs=(DistributedFileSystem)cluster.getFileSystem();
    dfs.mkdirs(path);
    dfs.allowSnapshot(path);
    dfs.createSnapshot(path);
    MBeanServer mbs=ManagementFactory.getPlatformMBeanServer();
    ObjectName mxbeanName=new ObjectName("Hadoop:service=NameNode,name=SnapshotInfo");
    CompositeData[] directories=(CompositeData[])mbs.getAttribute(mxbeanName,"SnapshottableDirectories");
    int numDirectories=Array.getLength(directories);
    assertEquals(sm.getNumSnapshottableDirs(),numDirectories);
    CompositeData[] snapshots=(CompositeData[])mbs.getAttribute(mxbeanName,"Snapshots");
    int numSnapshots=Array.getLength(snapshots);
    assertEquals(sm.getNumSnapshots(),numSnapshots);
    CompositeData d=(CompositeData)Array.get(directories,0);
    CompositeData s=(CompositeData)Array.get(snapshots,0);
    assertTrue(((String)d.get("path")).contains(pathName));
    assertTrue(((String)s.get("snapshotDirectory")).contains(pathName));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestSnapshottableDirListing

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test the listing with different user names to make sure only directories
 * that are owned by the user are listed.
 */
@Test(timeout=60000) public void testListWithDifferentUser() throws Exception {
  cluster.getNamesystem().getSnapshotManager().setAllowNestedSnapshots(true);
  hdfs.allowSnapshot(dir1);
  hdfs.allowSnapshot(dir2);
  hdfs.setPermission(root,FsPermission.valueOf("-rwxrwxrwx"));
  UserGroupInformation ugi1=UserGroupInformation.createUserForTesting("user1",new String[]{"group1"});
  DistributedFileSystem fs1=(DistributedFileSystem)DFSTestUtil.getFileSystemAs(ugi1,conf);
  Path dir1_user1=new Path("/dir1_user1");
  Path dir2_user1=new Path("/dir2_user1");
  fs1.mkdirs(dir1_user1);
  fs1.mkdirs(dir2_user1);
  hdfs.allowSnapshot(dir1_user1);
  hdfs.allowSnapshot(dir2_user1);
  UserGroupInformation ugi2=UserGroupInformation.createUserForTesting("user2",new String[]{"group2"});
  DistributedFileSystem fs2=(DistributedFileSystem)DFSTestUtil.getFileSystemAs(ugi2,conf);
  Path dir_user2=new Path("/dir_user2");
  Path subdir_user2=new Path(dir_user2,"subdir");
  fs2.mkdirs(dir_user2);
  fs2.mkdirs(subdir_user2);
  hdfs.allowSnapshot(dir_user2);
  hdfs.allowSnapshot(subdir_user2);
  String supergroup=conf.get(DFS_PERMISSIONS_SUPERUSERGROUP_KEY,DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT);
  UserGroupInformation superUgi=UserGroupInformation.createUserForTesting("superuser",new String[]{supergroup});
  DistributedFileSystem fs3=(DistributedFileSystem)DFSTestUtil.getFileSystemAs(superUgi,conf);
  SnapshottableDirectoryStatus[] dirs=fs3.getSnapshottableDirListing();
  assertEquals(6,dirs.length);
  dirs=fs1.getSnapshottableDirListing();
  assertEquals(2,dirs.length);
  assertEquals(dir1_user1,dirs[0].getFullPath());
  assertEquals(dir2_user1,dirs[1].getFullPath());
  dirs=fs2.getSnapshottableDirListing();
  assertEquals(2,dirs.length);
  assertEquals(dir_user2,dirs[0].getFullPath());
  assertEquals(subdir_user2,dirs[1].getFullPath());
}

Class: org.apache.hadoop.hdfs.server.namenode.snapshot.TestXAttrWithSnapshot

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test that users can copy a snapshot while preserving its xattrs.
 */
@Test(timeout=120000) public void testCopySnapshotShouldPreserveXAttrs() throws Exception {
  FileSystem.mkdirs(hdfs,path,FsPermission.createImmutable((short)0700));
  hdfs.setXAttr(path,name1,value1);
  hdfs.setXAttr(path,name2,value2);
  SnapshotTestHelper.createSnapshot(hdfs,path,snapshotName);
  Path snapshotCopy=new Path(path.toString() + "-copy");
  String[] argv=new String[]{"-cp","-px",snapshotPath.toUri().toString(),snapshotCopy.toUri().toString()};
  int ret=ToolRunner.run(new FsShell(conf),argv);
  assertEquals("cp -px is not working on a snapshot",SUCCESS,ret);
  Map xattrs=hdfs.getXAttrs(snapshotCopy);
  assertArrayEquals(value1,xattrs.get(name1));
  assertArrayEquals(value2,xattrs.get(name2));
}

Class: org.apache.hadoop.hdfs.server.namenode.startupprogress.TestStartupProgress

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test(timeout=10000) public void testThreadSafety() throws Exception {
  int numThreads=100;
  Phase[] phases={LOADING_FSIMAGE,LOADING_FSIMAGE,LOADING_EDITS,LOADING_EDITS};
  Step[] steps=new Step[]{new Step(INODES),new Step(DELEGATION_KEYS),new Step(INODES),new Step(DELEGATION_KEYS)};
  String[] files={"file1","file1","file2","file2"};
  long[] sizes={1000L,1000L,2000L,2000L};
  long[] totals={10000L,20000L,30000L,40000L};
  ExecutorService exec=Executors.newFixedThreadPool(numThreads);
  try {
    for (int i=0; i < numThreads; ++i) {
      final Phase phase=phases[i % phases.length];
      final Step step=steps[i % steps.length];
      final String file=files[i % files.length];
      final long size=sizes[i % sizes.length];
      final long total=totals[i % totals.length];
      exec.submit(new Callable(){
        @Override public Void call(){
          startupProgress.beginPhase(phase);
          startupProgress.setFile(phase,file);
          startupProgress.setSize(phase,size);
          startupProgress.setTotal(phase,step,total);
          incrementCounter(startupProgress,phase,step,100L);
          startupProgress.endStep(phase,step);
          startupProgress.endPhase(phase);
          return null;
        }
      }
);
    }
  }
  finally {
    exec.shutdown();
    assertTrue(exec.awaitTermination(10000L,TimeUnit.MILLISECONDS));
  }
  StartupProgressView view=startupProgress.createView();
  assertNotNull(view);
  assertEquals("file1",view.getFile(LOADING_FSIMAGE));
  assertEquals(1000L,view.getSize(LOADING_FSIMAGE));
  assertEquals(10000L,view.getTotal(LOADING_FSIMAGE,new Step(INODES)));
  assertEquals(2500L,view.getCount(LOADING_FSIMAGE,new Step(INODES)));
  assertEquals(20000L,view.getTotal(LOADING_FSIMAGE,new Step(DELEGATION_KEYS)));
  assertEquals(2500L,view.getCount(LOADING_FSIMAGE,new Step(DELEGATION_KEYS)));
  assertEquals("file2",view.getFile(LOADING_EDITS));
  assertEquals(2000L,view.getSize(LOADING_EDITS));
  assertEquals(30000L,view.getTotal(LOADING_EDITS,new Step(INODES)));
  assertEquals(2500L,view.getCount(LOADING_EDITS,new Step(INODES)));
  assertEquals(40000L,view.getTotal(LOADING_EDITS,new Step(DELEGATION_KEYS)));
  assertEquals(2500L,view.getCount(LOADING_EDITS,new Step(DELEGATION_KEYS)));
}

Class: org.apache.hadoop.hdfs.server.namenode.startupprogress.TestStartupProgressMetrics

APIUtilityVerifier BooleanVerifier 

@Test public void testRunningState(){
  setStartupProgressForRunningState(startupProgress);
  MetricsRecordBuilder builder=getMetrics(metrics,true);
  assertTrue(getLongCounter("ElapsedTime",builder) >= 0L);
  assertGauge("PercentComplete",0.375f,builder);
  assertCounter("LoadingFsImageCount",100L,builder);
  assertTrue(getLongCounter("LoadingFsImageElapsedTime",builder) >= 0L);
  assertCounter("LoadingFsImageTotal",100L,builder);
  assertGauge("LoadingFsImagePercentComplete",1.0f,builder);
  assertCounter("LoadingEditsCount",100L,builder);
  assertTrue(getLongCounter("LoadingEditsElapsedTime",builder) >= 0L);
  assertCounter("LoadingEditsTotal",200L,builder);
  assertGauge("LoadingEditsPercentComplete",0.5f,builder);
  assertCounter("SavingCheckpointCount",0L,builder);
  assertCounter("SavingCheckpointElapsedTime",0L,builder);
  assertCounter("SavingCheckpointTotal",0L,builder);
  assertGauge("SavingCheckpointPercentComplete",0.0f,builder);
  assertCounter("SafeModeCount",0L,builder);
  assertCounter("SafeModeElapsedTime",0L,builder);
  assertCounter("SafeModeTotal",0L,builder);
  assertGauge("SafeModePercentComplete",0.0f,builder);
}

APIUtilityVerifier BooleanVerifier 

@Test public void testFinalState(){
  setStartupProgressForFinalState(startupProgress);
  MetricsRecordBuilder builder=getMetrics(metrics,true);
  assertTrue(getLongCounter("ElapsedTime",builder) >= 0L);
  assertGauge("PercentComplete",1.0f,builder);
  assertCounter("LoadingFsImageCount",100L,builder);
  assertTrue(getLongCounter("LoadingFsImageElapsedTime",builder) >= 0L);
  assertCounter("LoadingFsImageTotal",100L,builder);
  assertGauge("LoadingFsImagePercentComplete",1.0f,builder);
  assertCounter("LoadingEditsCount",200L,builder);
  assertTrue(getLongCounter("LoadingEditsElapsedTime",builder) >= 0L);
  assertCounter("LoadingEditsTotal",200L,builder);
  assertGauge("LoadingEditsPercentComplete",1.0f,builder);
  assertCounter("SavingCheckpointCount",300L,builder);
  assertTrue(getLongCounter("SavingCheckpointElapsedTime",builder) >= 0L);
  assertCounter("SavingCheckpointTotal",300L,builder);
  assertGauge("SavingCheckpointPercentComplete",1.0f,builder);
  assertCounter("SafeModeCount",400L,builder);
  assertTrue(getLongCounter("SafeModeElapsedTime",builder) >= 0L);
  assertCounter("SafeModeTotal",400L,builder);
  assertGauge("SafeModePercentComplete",1.0f,builder);
}

Class: org.apache.hadoop.hdfs.server.namenode.web.resources.TestWebHdfsDataLocality

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier 

@Test public void testDataLocality() throws Exception {
  final Configuration conf=WebHdfsTestUtil.createConf();
  final String[] racks={RACK0,RACK0,RACK1,RACK1,RACK2,RACK2};
  final int nDataNodes=racks.length;
  LOG.info("nDataNodes=" + nDataNodes + ", racks="+ Arrays.asList(racks));
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(nDataNodes).racks(racks).build();
  try {
    cluster.waitActive();
    final DistributedFileSystem dfs=cluster.getFileSystem();
    final NameNode namenode=cluster.getNameNode();
    final DatanodeManager dm=namenode.getNamesystem().getBlockManager().getDatanodeManager();
    LOG.info("dm=" + dm);
    final long blocksize=DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
    final String f="/foo";
{
      for (int i=0; i < nDataNodes; i++) {
        final DataNode dn=cluster.getDataNodes().get(i);
        final String ipAddr=dm.getDatanode(dn.getDatanodeId()).getIpAddr();
        final DatanodeInfo chosen=NamenodeWebHdfsMethods.chooseDatanode(namenode,f,PutOpParam.Op.CREATE,-1L,blocksize,null);
        Assert.assertEquals(ipAddr,chosen.getIpAddr());
      }
    }
    final Path p=new Path(f);
    final FSDataOutputStream out=dfs.create(p,(short)1);
    out.write(1);
    out.close();
    final LocatedBlocks locatedblocks=NameNodeAdapter.getBlockLocations(namenode,f,0,1);
    final List lb=locatedblocks.getLocatedBlocks();
    Assert.assertEquals(1,lb.size());
    final DatanodeInfo[] locations=lb.get(0).getLocations();
    Assert.assertEquals(1,locations.length);
    final DatanodeInfo expected=locations[0];
{
      final DatanodeInfo chosen=NamenodeWebHdfsMethods.chooseDatanode(namenode,f,GetOpParam.Op.GETFILECHECKSUM,-1L,blocksize,null);
      Assert.assertEquals(expected,chosen);
    }
{
      final DatanodeInfo chosen=NamenodeWebHdfsMethods.chooseDatanode(namenode,f,GetOpParam.Op.OPEN,0,blocksize,null);
      Assert.assertEquals(expected,chosen);
    }
{
      final DatanodeInfo chosen=NamenodeWebHdfsMethods.chooseDatanode(namenode,f,PostOpParam.Op.APPEND,-1L,blocksize,null);
      Assert.assertEquals(expected,chosen);
    }
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier PublicFieldVerifier 

@Test public void testExcludeDataNodes() throws Exception {
  final Configuration conf=WebHdfsTestUtil.createConf();
  final String[] racks={RACK0,RACK0,RACK1,RACK1,RACK2,RACK2};
  final String[] hosts={"DataNode1","DataNode2","DataNode3","DataNode4","DataNode5","DataNode6"};
  final int nDataNodes=hosts.length;
  LOG.info("nDataNodes=" + nDataNodes + ", racks="+ Arrays.asList(racks)+ ", hosts="+ Arrays.asList(hosts));
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).hosts(hosts).numDataNodes(nDataNodes).racks(racks).build();
  try {
    cluster.waitActive();
    final DistributedFileSystem dfs=cluster.getFileSystem();
    final NameNode namenode=cluster.getNameNode();
    final DatanodeManager dm=namenode.getNamesystem().getBlockManager().getDatanodeManager();
    LOG.info("dm=" + dm);
    final long blocksize=DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
    final String f="/foo";
    final Path p=new Path(f);
    final FSDataOutputStream out=dfs.create(p,(short)3);
    out.write(1);
    out.close();
    final LocatedBlocks locatedblocks=NameNodeAdapter.getBlockLocations(namenode,f,0,1);
    final List lb=locatedblocks.getLocatedBlocks();
    Assert.assertEquals(1,lb.size());
    final DatanodeInfo[] locations=lb.get(0).getLocations();
    Assert.assertEquals(3,locations.length);
    StringBuffer sb=new StringBuffer();
    for (int i=0; i < 2; i++) {
      sb.append(locations[i].getXferAddr());
{
        final DatanodeInfo chosen=NamenodeWebHdfsMethods.chooseDatanode(namenode,f,GetOpParam.Op.GETFILECHECKSUM,-1L,blocksize,sb.toString());
        for (int j=0; j <= i; j++) {
          Assert.assertNotEquals(locations[j].getHostName(),chosen.getHostName());
        }
      }
{
        final DatanodeInfo chosen=NamenodeWebHdfsMethods.chooseDatanode(namenode,f,GetOpParam.Op.OPEN,0,blocksize,sb.toString());
        for (int j=0; j <= i; j++) {
          Assert.assertNotEquals(locations[j].getHostName(),chosen.getHostName());
        }
      }
{
        final DatanodeInfo chosen=NamenodeWebHdfsMethods.chooseDatanode(namenode,f,PostOpParam.Op.APPEND,-1L,blocksize,sb.toString());
        for (int j=0; j <= i; j++) {
          Assert.assertNotEquals(locations[j].getHostName(),chosen.getHostName());
        }
      }
      sb.append(",");
    }
  }
  finally {
    cluster.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.shortcircuit.TestShortCircuitCache

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier HybridVerifier 

@Test(timeout=60000) public void testShmBasedStaleness() throws Exception {
  BlockReaderTestUtil.enableShortCircuitShmTracing();
  TemporarySocketDirectory sockDir=new TemporarySocketDirectory();
  Configuration conf=createShortCircuitConf("testShmBasedStaleness",sockDir);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  DistributedFileSystem fs=cluster.getFileSystem();
  final ShortCircuitCache cache=fs.getClient().getClientContext().getShortCircuitCache();
  String TEST_FILE="/test_file";
  final int TEST_FILE_LEN=8193;
  final int SEED=0xFADED;
  DFSTestUtil.createFile(fs,new Path(TEST_FILE),TEST_FILE_LEN,(short)1,SEED);
  FSDataInputStream fis=fs.open(new Path(TEST_FILE));
  int first=fis.read();
  final ExtendedBlock block=DFSTestUtil.getFirstBlock(fs,new Path(TEST_FILE));
  Assert.assertTrue(first != -1);
  cache.accept(new CacheVisitor(){
    @Override public void visit(    int numOutstandingMmaps,    Map replicas,    Map failedLoads,    Map evictable,    Map evictableMmapped){
      ShortCircuitReplica replica=replicas.get(ExtendedBlockId.fromExtendedBlock(block));
      Assert.assertNotNull(replica);
      Assert.assertTrue(replica.getSlot().isValid());
    }
  }
);
  cluster.getDataNodes().get(0).shutdown();
  cache.accept(new CacheVisitor(){
    @Override public void visit(    int numOutstandingMmaps,    Map replicas,    Map failedLoads,    Map evictable,    Map evictableMmapped){
      ShortCircuitReplica replica=replicas.get(ExtendedBlockId.fromExtendedBlock(block));
      Assert.assertNotNull(replica);
      Assert.assertFalse(replica.getSlot().isValid());
    }
  }
);
  cluster.shutdown();
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test unlinking a file whose blocks we are caching in the DFSClient.
 * The DataNode will notify the DFSClient that the replica is stale via the
 * ShortCircuitShm.
 */
@Test(timeout=60000) public void testUnlinkingReplicasInFileDescriptorCache() throws Exception {
  BlockReaderTestUtil.enableShortCircuitShmTracing();
  TemporarySocketDirectory sockDir=new TemporarySocketDirectory();
  Configuration conf=createShortCircuitConf("testUnlinkingReplicasInFileDescriptorCache",sockDir);
  conf.setLong(DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY,1000000000L);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  DistributedFileSystem fs=cluster.getFileSystem();
  final ShortCircuitCache cache=fs.getClient().getClientContext().getShortCircuitCache();
  cache.getDfsClientShmManager().visit(new Visitor(){
    @Override public void visit(    HashMap info) throws IOException {
      Assert.assertEquals(0,info.size());
    }
  }
);
  final Path TEST_PATH=new Path("/test_file");
  final int TEST_FILE_LEN=8193;
  final int SEED=0xFADE0;
  DFSTestUtil.createFile(fs,TEST_PATH,TEST_FILE_LEN,(short)1,SEED);
  byte contents[]=DFSTestUtil.readFileBuffer(fs,TEST_PATH);
  byte expected[]=DFSTestUtil.calculateFileContentsFromSeed(SEED,TEST_FILE_LEN);
  Assert.assertTrue(Arrays.equals(contents,expected));
  final DatanodeInfo datanode=new DatanodeInfo(cluster.getDataNodes().get(0).getDatanodeId());
  cache.getDfsClientShmManager().visit(new Visitor(){
    @Override public void visit(    HashMap info) throws IOException {
      Assert.assertTrue(info.get(datanode).full.isEmpty());
      Assert.assertFalse(info.get(datanode).disabled);
      Assert.assertEquals(1,info.get(datanode).notFull.values().size());
      DfsClientShm shm=info.get(datanode).notFull.values().iterator().next();
      Assert.assertFalse(shm.isDisconnected());
    }
  }
);
  fs.delete(TEST_PATH,false);
  GenericTestUtils.waitFor(new Supplier(){
    MutableBoolean done=new MutableBoolean(true);
    @Override public Boolean get(){
      try {
        done.setValue(true);
        cache.getDfsClientShmManager().visit(new Visitor(){
          @Override public void visit(          HashMap info) throws IOException {
            Assert.assertTrue(info.get(datanode).full.isEmpty());
            Assert.assertFalse(info.get(datanode).disabled);
            Assert.assertEquals(1,info.get(datanode).notFull.values().size());
            DfsClientShm shm=info.get(datanode).notFull.values().iterator().next();
            for (Iterator iter=shm.slotIterator(); iter.hasNext(); ) {
              Slot slot=iter.next();
              if (slot.isValid()) {
                done.setValue(false);
              }
            }
          }
        }
);
      }
 catch (      IOException e) {
        LOG.error("error running visitor",e);
      }
      return done.booleanValue();
    }
  }
,10,60000);
  cluster.shutdown();
  sockDir.close();
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test(timeout=60000) public void testAllocShm() throws Exception {
  BlockReaderTestUtil.enableShortCircuitShmTracing();
  TemporarySocketDirectory sockDir=new TemporarySocketDirectory();
  Configuration conf=createShortCircuitConf("testAllocShm",sockDir);
  MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  cluster.waitActive();
  DistributedFileSystem fs=cluster.getFileSystem();
  final ShortCircuitCache cache=fs.getClient().getClientContext().getShortCircuitCache();
  cache.getDfsClientShmManager().visit(new Visitor(){
    @Override public void visit(    HashMap info) throws IOException {
      Assert.assertEquals(0,info.size());
    }
  }
);
  DomainPeer peer=getDomainPeerToDn(conf);
  MutableBoolean usedPeer=new MutableBoolean(false);
  ExtendedBlockId blockId=new ExtendedBlockId(123,"xyz");
  final DatanodeInfo datanode=new DatanodeInfo(cluster.getDataNodes().get(0).getDatanodeId());
  Slot slot=cache.allocShmSlot(datanode,peer,usedPeer,blockId,"testAllocShm_client");
  Assert.assertNotNull(slot);
  Assert.assertTrue(usedPeer.booleanValue());
  cache.getDfsClientShmManager().visit(new Visitor(){
    @Override public void visit(    HashMap info) throws IOException {
      Assert.assertEquals(1,info.size());
      PerDatanodeVisitorInfo vinfo=info.get(datanode);
      Assert.assertFalse(vinfo.disabled);
      Assert.assertEquals(0,vinfo.full.size());
      Assert.assertEquals(1,vinfo.notFull.size());
    }
  }
);
  cache.scheduleSlotReleaser(slot);
  GenericTestUtils.waitFor(new Supplier(){
    @Override public Boolean get(){
      final MutableBoolean done=new MutableBoolean(false);
      try {
        cache.getDfsClientShmManager().visit(new Visitor(){
          @Override public void visit(          HashMap info) throws IOException {
            done.setValue(info.get(datanode).full.isEmpty() && info.get(datanode).notFull.isEmpty());
          }
        }
);
      }
 catch (      IOException e) {
        LOG.error("error running visitor",e);
      }
      return done.booleanValue();
    }
  }
,10,60000);
  cluster.shutdown();
  sockDir.close();
}

Class: org.apache.hadoop.hdfs.shortcircuit.TestShortCircuitShm

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=60000) public void testAllocateSlots() throws Exception {
  File path=new File(TEST_BASE,"testAllocateSlots");
  path.mkdirs();
  SharedFileDescriptorFactory factory=SharedFileDescriptorFactory.create("shm_",new String[]{path.getAbsolutePath()});
  FileInputStream stream=factory.createDescriptor("testAllocateSlots",4096);
  ShortCircuitShm shm=new ShortCircuitShm(ShmId.createRandom(),stream);
  int numSlots=0;
  ArrayList slots=new ArrayList();
  while (!shm.isFull()) {
    Slot slot=shm.allocAndRegisterSlot(new ExtendedBlockId(123L,"test_bp1"));
    slots.add(slot);
    numSlots++;
  }
  LOG.info("allocated " + numSlots + " slots before running out.");
  int slotIdx=0;
  for (Iterator iter=shm.slotIterator(); iter.hasNext(); ) {
    Assert.assertTrue(slots.contains(iter.next()));
  }
  for (  Slot slot : slots) {
    Assert.assertFalse(slot.addAnchor());
    Assert.assertEquals(slotIdx++,slot.getSlotIdx());
  }
  for (  Slot slot : slots) {
    slot.makeAnchorable();
  }
  for (  Slot slot : slots) {
    Assert.assertTrue(slot.addAnchor());
  }
  for (  Slot slot : slots) {
    slot.removeAnchor();
  }
  for (  Slot slot : slots) {
    shm.unregisterSlot(slot.getSlotIdx());
    slot.makeInvalid();
  }
  shm.free();
  stream.close();
  FileUtil.fullyDelete(path);
}

Class: org.apache.hadoop.hdfs.tools.TestDFSHAAdminMiniCluster

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test failover with various options
 */
@Test public void testFencer() throws Exception {
  assertEquals(-1,runTool("-failover","nn1","nn2"));
  File tmpFile=File.createTempFile("testFencer",".txt");
  tmpFile.deleteOnExit();
  if (Shell.WINDOWS) {
    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,"shell(echo %target_nameserviceid%.%target_namenodeid% " + "%target_port% %dfs_ha_namenode_id% > " + tmpFile.getAbsolutePath() + ")");
  }
 else {
    conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,"shell(echo -n $target_nameserviceid.$target_namenodeid " + "$target_port $dfs_ha_namenode_id > " + tmpFile.getAbsolutePath() + ")");
  }
  tool.setConf(conf);
  assertEquals(0,runTool("-transitionToActive","nn1"));
  assertEquals(0,runTool("-failover","nn1","nn2"));
  assertEquals(0,runTool("-ns","minidfs-ns","-failover","nn2","nn1"));
  assertEquals("",Files.toString(tmpFile,Charsets.UTF_8));
  assertEquals(0,runTool("-failover","nn1","nn2","--forcefence"));
  String fenceCommandOutput=Files.toString(tmpFile,Charsets.UTF_8).replaceAll(" *[\r\n]+","");
  assertEquals("minidfs-ns.nn1 " + nn1Port + " nn1",fenceCommandOutput);
  tmpFile.delete();
  assertEquals(0,runTool("-failover","nn2","nn1","--forceactive"));
  assertFalse(tmpFile.exists());
  conf.unset(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY);
  tool.setConf(conf);
  assertEquals(-1,runTool("-failover","nn1","nn2","--forcefence"));
  assertFalse(tmpFile.exists());
  conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,"foobar!");
  tool.setConf(conf);
  assertEquals(-1,runTool("-failover","nn1","nn2","--forcefence"));
  assertFalse(tmpFile.exists());
  conf.set(DFSConfigKeys.DFS_HA_FENCE_METHODS_KEY,TestDFSHAAdmin.getFencerTrueCommand());
  tool.setConf(conf);
  assertEquals(0,runTool("-failover","--forcefence","nn1","nn2"));
}

Class: org.apache.hadoop.hdfs.tools.TestDelegationTokenFetcher

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Call fetch token using http server
 */
@Test public void expectedTokenIsRetrievedFromHttp() throws Exception {
  final Token testToken=new Token("id".getBytes(),"pwd".getBytes(),FakeRenewer.KIND,new Text("127.0.0.1:1234"));
  WebHdfsFileSystem fs=mock(WebHdfsFileSystem.class);
  doReturn(testToken).when(fs).getDelegationToken(anyString());
  Path p=new Path(f.getRoot().getAbsolutePath(),tokenFile);
  DelegationTokenFetcher.saveDelegationToken(conf,fs,null,p);
  Credentials creds=Credentials.readTokenStorageFile(p,conf);
  Iterator> itr=creds.getAllTokens().iterator();
  assertTrue("token not exist error",itr.hasNext());
  Token fetchedToken=itr.next();
  Assert.assertArrayEquals("token wrong identifier error",testToken.getIdentifier(),fetchedToken.getIdentifier());
  Assert.assertArrayEquals("token wrong password error",testToken.getPassword(),fetchedToken.getPassword());
  DelegationTokenFetcher.renewTokens(conf,p);
  Assert.assertEquals(testToken,FakeRenewer.getLastRenewed());
  DelegationTokenFetcher.cancelTokens(conf,p);
  Assert.assertEquals(testToken,FakeRenewer.getLastCanceled());
}

Class: org.apache.hadoop.hdfs.tools.TestGetConf

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void TestGetConfExcludeCommand() throws Exception {
  HdfsConfiguration conf=new HdfsConfiguration();
  localFileSys=FileSystem.getLocal(conf);
  Path workingDir=localFileSys.getWorkingDirectory();
  Path dir=new Path(workingDir,System.getProperty("test.build.data","target/test/data") + "/Getconf/");
  Path hostsFile=new Path(dir,"hosts");
  Path excludeFile=new Path(dir,"exclude");
  conf.set(DFSConfigKeys.DFS_HOSTS,hostsFile.toUri().getPath());
  conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE,excludeFile.toUri().getPath());
  writeConfigFile(hostsFile,null);
  writeConfigFile(excludeFile,null);
  String[] args={"-excludeFile"};
  String ret=runTool(conf,args,true);
  assertEquals(excludeFile.toUri().getPath(),ret.trim());
  cleanupFile(localFileSys,excludeFile.getParent());
}

APIUtilityVerifier BooleanVerifier 

/** 
 * Test invalid argument to the tool
 */
@Test(timeout=10000) public void testInvalidArgument() throws Exception {
  HdfsConfiguration conf=new HdfsConfiguration();
  String[] args={"-invalidArgument"};
  String ret=runTool(conf,args,false);
  assertTrue(ret.contains(GetConf.USAGE));
}

APIUtilityVerifier InternalCallVerifier NullVerifier 

/** 
 * Test empty configuration
 */
@Test(timeout=10000) public void testEmptyConf() throws Exception {
  HdfsConfiguration conf=new HdfsConfiguration(false);
  getAddressListFromTool(TestType.NAMENODE,conf,false);
  System.out.println(getAddressListFromTool(TestType.BACKUP,conf,false));
  getAddressListFromTool(TestType.SECONDARY,conf,false);
  getAddressListFromTool(TestType.NNRPCADDRESSES,conf,false);
  for (  Command cmd : Command.values()) {
    String arg=cmd.getName();
    CommandHandler handler=Command.getHandler(arg);
    assertNotNull("missing handler: " + cmd,handler);
    if (handler.key != null) {
      String[] args={handler.key};
      runTool(conf,args,false);
    }
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=10000) public void testGetSpecificKey() throws Exception {
  HdfsConfiguration conf=new HdfsConfiguration();
  conf.set("mykey"," myval ");
  String[] args={"-confKey","mykey"};
  String toolResult=runTool(conf,args,true);
  assertEquals(String.format("myval%n"),toolResult);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void TestGetConfIncludeCommand() throws Exception {
  HdfsConfiguration conf=new HdfsConfiguration();
  localFileSys=FileSystem.getLocal(conf);
  Path workingDir=localFileSys.getWorkingDirectory();
  Path dir=new Path(workingDir,System.getProperty("test.build.data","target/test/data") + "/Getconf/");
  Path hostsFile=new Path(dir,"hosts");
  Path excludeFile=new Path(dir,"exclude");
  conf.set(DFSConfigKeys.DFS_HOSTS,hostsFile.toUri().getPath());
  conf.set(DFSConfigKeys.DFS_HOSTS_EXCLUDE,excludeFile.toUri().getPath());
  writeConfigFile(hostsFile,null);
  writeConfigFile(excludeFile,null);
  String[] args={"-includeFile"};
  String ret=runTool(conf,args,true);
  assertEquals(hostsFile.toUri().getPath(),ret.trim());
  cleanupFile(localFileSys,excludeFile.getParent());
}

Class: org.apache.hadoop.hdfs.tools.offlineImageViewer.TestOfflineImageViewer

APIUtilityVerifier EqualityVerifier 

@Test public void testFileDistributionCalculatorWithOptions() throws IOException {
  int status=OfflineImageViewerPB.run(new String[]{"-i",originalFsimage.getAbsolutePath(),"-o","-","-p","FileDistribution","-maxSize","512","-step","8"});
  assertEquals(0,status);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testWebImageViewer() throws IOException, InterruptedException, URISyntaxException {
  WebImageViewer viewer=new WebImageViewer(NetUtils.createSocketAddr("localhost:0"));
  try {
    viewer.initServer(originalFsimage.getAbsolutePath());
    int port=viewer.getPort();
    URI uri=new URI("webhdfs://localhost:" + String.valueOf(port));
    Configuration conf=new Configuration();
    WebHdfsFileSystem webhdfs=(WebHdfsFileSystem)FileSystem.get(uri,conf);
    FileStatus[] statuses=webhdfs.listStatus(new Path("/"));
    assertEquals(NUM_DIRS + 2,statuses.length);
    statuses=webhdfs.listStatus(new Path("/dir0"));
    assertEquals(FILES_PER_DIR,statuses.length);
    FileStatus status=webhdfs.listStatus(new Path("/dir0/file0"))[0];
    FileStatus expected=writtenFiles.get("/dir0/file0");
    compareFile(expected,status);
    statuses=webhdfs.listStatus(new Path("/emptydir"));
    assertEquals(0,statuses.length);
    URL url=new URL("http://localhost:" + port + "/webhdfs/v1/invalid/?op=LISTSTATUS");
    verifyHttpResponseCode(HttpURLConnection.HTTP_NOT_FOUND,url);
    url=new URL("http://localhost:" + port + "/webhdfs/v1?op=LISTSTATUS");
    verifyHttpResponseCode(HttpURLConnection.HTTP_NOT_FOUND,url);
    status=webhdfs.getFileStatus(new Path("/dir0/file0"));
    compareFile(expected,status);
    url=new URL("http://localhost:" + port + "/webhdfs/v1/invalid/?op=GETFILESTATUS");
    verifyHttpResponseCode(HttpURLConnection.HTTP_NOT_FOUND,url);
    url=new URL("http://localhost:" + port + "/webhdfs/v1/?op=INVALID");
    verifyHttpResponseCode(HttpURLConnection.HTTP_BAD_REQUEST,url);
    url=new URL("http://localhost:" + port + "/webhdfs/v1/?op=LISTSTATUS");
    HttpURLConnection connection=(HttpURLConnection)url.openConnection();
    connection.setRequestMethod("POST");
    connection.connect();
    assertEquals(HttpURLConnection.HTTP_BAD_METHOD,connection.getResponseCode());
  }
  finally {
    viewer.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testFileDistributionCalculator() throws IOException {
  StringWriter output=new StringWriter();
  PrintWriter o=new PrintWriter(output);
  new FileDistributionCalculator(new Configuration(),0,0,o).visit(new RandomAccessFile(originalFsimage,"r"));
  o.close();
  Pattern p=Pattern.compile("totalFiles = (\\d+)\n");
  Matcher matcher=p.matcher(output.getBuffer());
  assertTrue(matcher.find() && matcher.groupCount() == 1);
  int totalFiles=Integer.parseInt(matcher.group(1));
  assertEquals(NUM_DIRS * FILES_PER_DIR,totalFiles);
  p=Pattern.compile("totalDirectories = (\\d+)\n");
  matcher=p.matcher(output.getBuffer());
  assertTrue(matcher.find() && matcher.groupCount() == 1);
  int totalDirs=Integer.parseInt(matcher.group(1));
  assertEquals(NUM_DIRS + 3,totalDirs);
  FileStatus maxFile=Collections.max(writtenFiles.values(),new Comparator(){
    @Override public int compare(    FileStatus first,    FileStatus second){
      return first.getLen() < second.getLen() ? -1 : ((first.getLen() == second.getLen()) ? 0 : 1);
    }
  }
);
  p=Pattern.compile("maxFileSize = (\\d+)\n");
  matcher=p.matcher(output.getBuffer());
  assertTrue(matcher.find() && matcher.groupCount() == 1);
  assertEquals(maxFile.getLen(),Long.parseLong(matcher.group(1)));
}

Class: org.apache.hadoop.hdfs.tools.offlineImageViewer.TestOfflineImageViewerForAcl

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testWebImageViewerForAcl() throws IOException, InterruptedException, URISyntaxException {
  WebImageViewer viewer=new WebImageViewer(NetUtils.createSocketAddr("localhost:0"));
  try {
    viewer.initServer(originalFsimage.getAbsolutePath());
    int port=viewer.getPort();
    URI uri=new URI("webhdfs://localhost:" + String.valueOf(port));
    Configuration conf=new Configuration();
    WebHdfsFileSystem webhdfs=(WebHdfsFileSystem)FileSystem.get(uri,conf);
    AclStatus acl=webhdfs.getAclStatus(new Path("/dirWithNoAcl"));
    assertEquals(writtenAcls.get("/dirWithNoAcl"),acl);
    acl=webhdfs.getAclStatus(new Path("/dirWithDefaultAcl"));
    assertEquals(writtenAcls.get("/dirWithDefaultAcl"),acl);
    acl=webhdfs.getAclStatus(new Path("/noAcl"));
    assertEquals(writtenAcls.get("/noAcl"),acl);
    acl=webhdfs.getAclStatus(new Path("/withAcl"));
    assertEquals(writtenAcls.get("/withAcl"),acl);
    acl=webhdfs.getAclStatus(new Path("/withSeveralAcls"));
    assertEquals(writtenAcls.get("/withSeveralAcls"),acl);
    URL url=new URL("http://localhost:" + port + "/webhdfs/v1/invalid/?op=GETACLSTATUS");
    HttpURLConnection connection=(HttpURLConnection)url.openConnection();
    connection.setRequestMethod("GET");
    connection.connect();
    assertEquals(HttpURLConnection.HTTP_NOT_FOUND,connection.getResponseCode());
  }
  finally {
    viewer.shutdown();
  }
}

Class: org.apache.hadoop.hdfs.util.TestAtomicFileOutputStream

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test case where there is no existing file
 */
@Test public void testOverwriteFile() throws IOException {
  assertTrue("Creating empty dst file",DST_FILE.createNewFile());
  OutputStream fos=new AtomicFileOutputStream(DST_FILE);
  assertTrue("Empty file still exists",DST_FILE.exists());
  fos.write(TEST_STRING.getBytes());
  fos.flush();
  assertEquals("",DFSTestUtil.readFile(DST_FILE));
  fos.close();
  String readBackData=DFSTestUtil.readFile(DST_FILE);
  assertEquals(TEST_STRING,readBackData);
}

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

/** 
 * Test case where there is no existing file
 */
@Test public void testWriteNewFile() throws IOException {
  OutputStream fos=new AtomicFileOutputStream(DST_FILE);
  assertFalse(DST_FILE.exists());
  fos.write(TEST_STRING.getBytes());
  fos.flush();
  assertFalse(DST_FILE.exists());
  fos.close();
  assertTrue(DST_FILE.exists());
  String readBackData=DFSTestUtil.readFile(DST_FILE);
  assertEquals(TEST_STRING,readBackData);
}

Class: org.apache.hadoop.hdfs.util.TestBestEffortLongFile

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testGetSet() throws IOException {
  BestEffortLongFile f=new BestEffortLongFile(FILE,12345L);
  try {
    assertEquals(12345L,f.get());
    assertTrue(FILE.exists());
    Random r=new Random();
    for (int i=0; i < 100; i++) {
      long newVal=r.nextLong();
      f.set(newVal);
      assertEquals(newVal,f.get());
      BestEffortLongFile f2=new BestEffortLongFile(FILE,999L);
      try {
        assertEquals(newVal,f2.get());
      }
  finally {
        IOUtils.closeStream(f2);
      }
    }
  }
  finally {
    IOUtils.closeStream(f);
  }
}

Class: org.apache.hadoop.hdfs.util.TestLightWeightHashSet

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testMultiBasic(){
  LOG.info("Test multi element basic");
  for (  Integer i : list) {
    assertTrue(set.add(i));
  }
  assertEquals(list.size(),set.size());
  for (  Integer i : list) {
    assertTrue(set.contains(i));
  }
  for (  Integer i : list) {
    assertFalse(set.add(i));
  }
  for (  Integer i : list) {
    assertTrue(set.contains(i));
  }
  Iterator iter=set.iterator();
  int num=0;
  while (iter.hasNext()) {
    Integer next=iter.next();
    assertNotNull(next);
    assertTrue(list.contains(next));
    num++;
  }
  assertEquals(list.size(),num);
  LOG.info("Test multi element basic - DONE");
}

Class: org.apache.hadoop.hdfs.util.TestLightWeightLinkedSet

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=60000) public void testGetBookmarkReturnsBookmarkIterator(){
  LOG.info("Test getBookmark returns proper iterator");
  assertTrue(set.addAll(list));
  Iterator bookmark=set.getBookmark();
  assertEquals(bookmark.next(),list.get(0));
  final int numAdvance=list.size() / 2;
  for (int i=1; i < numAdvance; i++) {
    bookmark.next();
  }
  Iterator bookmark2=set.getBookmark();
  assertEquals(bookmark2.next(),list.get(numAdvance));
}

Class: org.apache.hadoop.hdfs.util.TestMD5FileUtils

APIUtilityVerifier EqualityVerifier 

@Test public void testComputeMd5ForFile() throws Exception {
  MD5Hash computedDigest=MD5FileUtils.computeMd5ForFile(TEST_FILE);
  assertEquals(TEST_MD5,computedDigest);
}

Class: org.apache.hadoop.hdfs.web.TestFSMainOperationsWebHdfs

APIUtilityVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testConcat() throws Exception {
  Path[] paths={new Path("/test/hadoop/file1"),new Path("/test/hadoop/file2"),new Path("/test/hadoop/file3")};
  DFSTestUtil.createFile(fSys,paths[0],1024,(short)3,0);
  DFSTestUtil.createFile(fSys,paths[1],1024,(short)3,0);
  DFSTestUtil.createFile(fSys,paths[2],1024,(short)3,0);
  Path catPath=new Path("/test/hadoop/catFile");
  DFSTestUtil.createFile(fSys,catPath,1024,(short)3,0);
  Assert.assertTrue(exists(fSys,catPath));
  fSys.concat(catPath,paths);
  Assert.assertFalse(exists(fSys,paths[0]));
  Assert.assertFalse(exists(fSys,paths[1]));
  Assert.assertFalse(exists(fSys,paths[2]));
  FileStatus fileStatus=fSys.getFileStatus(catPath);
  Assert.assertEquals(1024 * 4,fileStatus.getLen());
}

Class: org.apache.hadoop.hdfs.web.TestHttpsFileSystem

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testSWebHdfsFileSystem() throws Exception {
  FileSystem fs=WebHdfsTestUtil.getWebHdfsFileSystem(conf,"swebhdfs");
  final Path f=new Path("/testswebhdfs");
  FSDataOutputStream os=fs.create(f);
  os.write(23);
  os.close();
  Assert.assertTrue(fs.exists(f));
  InputStream is=fs.open(f);
  Assert.assertEquals(23,is.read());
  is.close();
  fs.close();
}

Class: org.apache.hadoop.hdfs.web.TestJsonUtil

APIUtilityVerifier IterativeVerifier EqualityVerifier 

@Test public void testToXAttrMap() throws IOException {
  String jsonString="{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"}," + "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}";
  Map json=(Map)JSON.parse(jsonString);
  XAttr xAttr1=(new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER).setName("a1").setValue(XAttrCodec.decodeValue("0x313233")).build();
  XAttr xAttr2=(new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER).setName("a2").setValue(XAttrCodec.decodeValue("0x313131")).build();
  List xAttrs=Lists.newArrayList();
  xAttrs.add(xAttr1);
  xAttrs.add(xAttr2);
  Map xAttrMap=XAttrHelper.buildXAttrMap(xAttrs);
  Map parsedXAttrMap=JsonUtil.toXAttrs(json);
  Assert.assertEquals(xAttrMap.size(),parsedXAttrMap.size());
  Iterator> iter=xAttrMap.entrySet().iterator();
  while (iter.hasNext()) {
    Entry entry=iter.next();
    Assert.assertArrayEquals(entry.getValue(),parsedXAttrMap.get(entry.getKey()));
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void testHdfsFileStatus(){
  final long now=Time.now();
  final String parent="/dir";
  final HdfsFileStatus status=new HdfsFileStatus(1001L,false,3,1L << 26,now,now + 10,new FsPermission((short)0644),"user","group",DFSUtil.string2Bytes("bar"),DFSUtil.string2Bytes("foo"),INodeId.GRANDFATHER_INODE_ID,0,null);
  final FileStatus fstatus=toFileStatus(status,parent);
  System.out.println("status  = " + status);
  System.out.println("fstatus = " + fstatus);
  final String json=JsonUtil.toJsonString(status,true);
  System.out.println("json    = " + json.replace(",",",\n  "));
  final HdfsFileStatus s2=JsonUtil.toFileStatus((Map)JSON.parse(json),true);
  final FileStatus fs2=toFileStatus(s2,parent);
  System.out.println("s2      = " + s2);
  System.out.println("fs2     = " + fs2);
  Assert.assertEquals(fstatus,fs2);
}

APIUtilityVerifier EqualityVerifier 

@Test public void testGetXAttrFromJson() throws IOException {
  String jsonString="{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"}," + "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}";
  Map json=(Map)JSON.parse(jsonString);
  byte[] value=JsonUtil.getXAttr(json,"user.a2");
  Assert.assertArrayEquals(XAttrCodec.decodeValue("0x313131"),value);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testToDatanodeInfoWithName() throws Exception {
  Map response=new HashMap();
  String name="127.0.0.1:1004";
  response.put("name",name);
  response.put("hostName","localhost");
  response.put("storageID","fake-id");
  response.put("infoPort",1338l);
  response.put("ipcPort",1339l);
  response.put("capacity",1024l);
  response.put("dfsUsed",512l);
  response.put("remaining",512l);
  response.put("blockPoolUsed",512l);
  response.put("lastUpdate",0l);
  response.put("xceiverCount",4096l);
  response.put("networkLocation","foo.bar.baz");
  response.put("adminState","NORMAL");
  response.put("cacheCapacity",123l);
  response.put("cacheUsed",321l);
  DatanodeInfo di=JsonUtil.toDatanodeInfo(response);
  Assert.assertEquals(name,di.getXferAddr());
  Map r=JsonUtil.toJsonMap(di);
  Assert.assertEquals(name,r.get("name"));
  Assert.assertEquals("127.0.0.1",r.get("ipAddr"));
  Assert.assertEquals(1004,(int)(Integer)r.get("xferPort"));
  String[] badNames={"127.0.0.1","127.0.0.1:",":","127.0.0.1:sweet",":123"};
  for (  String badName : badNames) {
    response.put("name",badName);
    checkDecodeFailure(response);
  }
  response.remove("name");
  checkDecodeFailure(response);
  response.put("ipAddr","127.0.0.1");
  checkDecodeFailure(response);
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testToAclStatus(){
  String jsonString="{\"AclStatus\":{\"entries\":[\"user::rwx\",\"user:user1:rw-\",\"group::rw-\",\"other::r-x\"],\"group\":\"supergroup\",\"owner\":\"testuser\",\"stickyBit\":false}}";
  Map json=(Map)JSON.parse(jsonString);
  List aclSpec=Lists.newArrayList(aclEntry(ACCESS,USER,ALL),aclEntry(ACCESS,USER,"user1",READ_WRITE),aclEntry(ACCESS,GROUP,READ_WRITE),aclEntry(ACCESS,OTHER,READ_EXECUTE));
  AclStatus.Builder aclStatusBuilder=new AclStatus.Builder();
  aclStatusBuilder.owner("testuser");
  aclStatusBuilder.group("supergroup");
  aclStatusBuilder.addEntries(aclSpec);
  aclStatusBuilder.stickyBit(false);
  Assert.assertEquals("Should be equal",aclStatusBuilder.build(),JsonUtil.toAclStatus(json));
}

Class: org.apache.hadoop.hdfs.web.TestTokenAspect

APIUtilityVerifier InternalCallVerifier BooleanVerifier IdentityVerifier HybridVerifier 

@Test public void testRenewal() throws Exception {
  Configuration conf=new Configuration();
  Token token1=mock(Token.class);
  Token token2=mock(Token.class);
  final long renewCycle=100;
  DelegationTokenRenewer.renewCycle=renewCycle;
  UserGroupInformation ugi=UserGroupInformation.createUserForTesting("foo",new String[]{"bar"});
  DummyFs fs=spy(new DummyFs());
  doReturn(token1).doReturn(token2).when(fs).getDelegationToken(null);
  doReturn(token1).when(fs).getRenewToken();
  doThrow(new IOException("renew failed")).when(token1).renew(conf);
  doThrow(new IOException("get failed")).when(fs).addDelegationTokens(null,null);
  final URI uri=new URI("dummyfs://127.0.0.1:1234");
  TokenAspect tokenAspect=new TokenAspect(fs,SecurityUtil.buildTokenService(uri),DummyFs.TOKEN_KIND);
  fs.initialize(uri,conf);
  tokenAspect.initDelegationToken(ugi);
  tokenAspect.ensureTokenInitialized();
  DelegationTokenRenewer.RenewAction action=getActionFromTokenAspect(tokenAspect);
  verify(fs).setDelegationToken(token1);
  assertTrue(action.isValid());
  Thread.sleep(renewCycle * 2);
  assertSame(action,getActionFromTokenAspect(tokenAspect));
  assertFalse(action.isValid());
  tokenAspect.ensureTokenInitialized();
  verify(fs,times(2)).getDelegationToken(anyString());
  verify(fs).setDelegationToken(token2);
  assertNotSame(action,getActionFromTokenAspect(tokenAspect));
  action=getActionFromTokenAspect(tokenAspect);
  assertTrue(action.isValid());
}

Class: org.apache.hadoop.hdfs.web.TestWebHDFS

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Test snapshot deletion through WebHdfs
 */
@Test public void testWebHdfsDeleteSnapshot() throws Exception {
  MiniDFSCluster cluster=null;
  final Configuration conf=WebHdfsTestUtil.createConf();
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    final DistributedFileSystem dfs=cluster.getFileSystem();
    final FileSystem webHdfs=WebHdfsTestUtil.getWebHdfsFileSystem(conf,WebHdfsFileSystem.SCHEME);
    final Path foo=new Path("/foo");
    dfs.mkdirs(foo);
    dfs.allowSnapshot(foo);
    webHdfs.createSnapshot(foo,"s1");
    final Path spath=webHdfs.createSnapshot(foo,null);
    Assert.assertTrue(webHdfs.exists(spath));
    final Path s1path=SnapshotTestHelper.getSnapshotRoot(foo,"s1");
    Assert.assertTrue(webHdfs.exists(s1path));
    webHdfs.deleteSnapshot(foo,"s1");
    Assert.assertFalse(webHdfs.exists(s1path));
    webHdfs.deleteSnapshot(foo,spath.getName());
    Assert.assertFalse(webHdfs.exists(spath));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test(timeout=300000) public void testLargeDirectory() throws Exception {
  final Configuration conf=WebHdfsTestUtil.createConf();
  final int listLimit=2;
  conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT,listLimit);
  FsPermission.setUMask(conf,new FsPermission((short)0077));
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
  try {
    cluster.waitActive();
    WebHdfsTestUtil.getWebHdfsFileSystem(conf,WebHdfsFileSystem.SCHEME).setPermission(new Path("/"),new FsPermission(FsAction.ALL,FsAction.ALL,FsAction.ALL));
    UserGroupInformation.setLoginUser(UserGroupInformation.createUserForTesting("not-superuser",new String[]{"not-supergroup"}));
    UserGroupInformation.createUserForTesting("me",new String[]{"my-group"}).doAs(new PrivilegedExceptionAction(){
      @Override public Void run() throws IOException, URISyntaxException {
        FileSystem fs=WebHdfsTestUtil.getWebHdfsFileSystem(conf,WebHdfsFileSystem.SCHEME);
        Path d=new Path("/my-dir");
        Assert.assertTrue(fs.mkdirs(d));
        for (int i=0; i < listLimit * 3; i++) {
          Path p=new Path(d,"file-" + i);
          Assert.assertTrue(fs.createNewFile(p));
        }
        Assert.assertEquals(listLimit * 3,fs.listStatus(d).length);
        return null;
      }
    }
);
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test(timeout=300000) public void testNumericalUserName() throws Exception {
  final Configuration conf=WebHdfsTestUtil.createConf();
  conf.set(DFSConfigKeys.DFS_WEBHDFS_USER_PATTERN_KEY,"^[A-Za-z0-9_][A-Za-z0-9._-]*[$]?$");
  final MiniDFSCluster cluster=new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    cluster.waitActive();
    WebHdfsTestUtil.getWebHdfsFileSystem(conf,WebHdfsFileSystem.SCHEME).setPermission(new Path("/"),new FsPermission(FsAction.ALL,FsAction.ALL,FsAction.ALL));
    UserGroupInformation.createUserForTesting("123",new String[]{"my-group"}).doAs(new PrivilegedExceptionAction(){
      @Override public Void run() throws IOException, URISyntaxException {
        FileSystem fs=WebHdfsTestUtil.getWebHdfsFileSystem(conf,WebHdfsFileSystem.SCHEME);
        Path d=new Path("/my-dir");
        Assert.assertTrue(fs.mkdirs(d));
        return null;
      }
    }
);
  }
  finally {
    cluster.shutdown();
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier HybridVerifier 

/** 
 * Test snapshot creation through WebHdfs
 */
@Test public void testWebHdfsCreateSnapshot() throws Exception {
  MiniDFSCluster cluster=null;
  final Configuration conf=WebHdfsTestUtil.createConf();
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    final DistributedFileSystem dfs=cluster.getFileSystem();
    final FileSystem webHdfs=WebHdfsTestUtil.getWebHdfsFileSystem(conf,WebHdfsFileSystem.SCHEME);
    final Path foo=new Path("/foo");
    dfs.mkdirs(foo);
    try {
      webHdfs.createSnapshot(foo);
      fail("Cannot create snapshot on a non-snapshottable directory");
    }
 catch (    Exception e) {
      GenericTestUtils.assertExceptionContains("Directory is not a snapshottable directory",e);
    }
    dfs.allowSnapshot(foo);
    webHdfs.createSnapshot(foo,"s1");
    final Path spath=webHdfs.createSnapshot(foo,null);
    Assert.assertTrue(webHdfs.exists(spath));
    final Path s1path=SnapshotTestHelper.getSnapshotRoot(foo,"s1");
    Assert.assertTrue(webHdfs.exists(s1path));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

/** 
 * Test snapshot rename through WebHdfs
 */
@Test public void testWebHdfsRenameSnapshot() throws Exception {
  MiniDFSCluster cluster=null;
  final Configuration conf=WebHdfsTestUtil.createConf();
  try {
    cluster=new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();
    final DistributedFileSystem dfs=cluster.getFileSystem();
    final FileSystem webHdfs=WebHdfsTestUtil.getWebHdfsFileSystem(conf,WebHdfsFileSystem.SCHEME);
    final Path foo=new Path("/foo");
    dfs.mkdirs(foo);
    dfs.allowSnapshot(foo);
    webHdfs.createSnapshot(foo,"s1");
    final Path s1path=SnapshotTestHelper.getSnapshotRoot(foo,"s1");
    Assert.assertTrue(webHdfs.exists(s1path));
    webHdfs.renameSnapshot(foo,"s1","s2");
    Assert.assertFalse(webHdfs.exists(s1path));
    final Path s2path=SnapshotTestHelper.getSnapshotRoot(foo,"s2");
    Assert.assertTrue(webHdfs.exists(s2path));
    webHdfs.deleteSnapshot(foo,"s2");
    Assert.assertFalse(webHdfs.exists(s2path));
  }
  finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.web.TestWebHDFSForHA

APIUtilityVerifier EqualityVerifier 

@Test public void testMultipleNamespacesConfigured() throws Exception {
  Configuration conf=DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
  MiniDFSCluster cluster=null;
  WebHdfsFileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).nnTopology(topo).numDataNodes(1).build();
    HATestUtil.setFailoverConfigurations(cluster,conf,LOGICAL_NAME);
    cluster.waitActive();
    DFSTestUtil.addHAConfiguration(conf,LOGICAL_NAME + "remote");
    DFSTestUtil.setFakeHttpAddresses(conf,LOGICAL_NAME + "remote");
    fs=(WebHdfsFileSystem)FileSystem.get(WEBHDFS_URI,conf);
    Assert.assertEquals(2,fs.getResolvedNNAddr().length);
  }
  finally {
    IOUtils.cleanup(null,fs);
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testHA() throws IOException {
  Configuration conf=DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
  MiniDFSCluster cluster=null;
  FileSystem fs=null;
  try {
    cluster=new MiniDFSCluster.Builder(conf).nnTopology(topo).numDataNodes(0).build();
    HATestUtil.setFailoverConfigurations(cluster,conf,LOGICAL_NAME);
    cluster.waitActive();
    fs=FileSystem.get(WEBHDFS_URI,conf);
    cluster.transitionToActive(0);
    final Path dir=new Path("/test");
    Assert.assertTrue(fs.mkdirs(dir));
    cluster.shutdownNameNode(0);
    cluster.transitionToActive(1);
    final Path dir2=new Path("/test2");
    Assert.assertTrue(fs.mkdirs(dir2));
  }
  finally {
    IOUtils.cleanup(null,fs);
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}

Class: org.apache.hadoop.hdfs.web.TestWebHdfsContentLength

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test public void testGetOpWithRedirect(){
  Future future1=contentLengthFuture(redirectResponse);
  Future future2=contentLengthFuture(errResponse);
  try {
    fs.open(p).read();
    Assert.fail();
  }
 catch (  IOException ioe) {
  }
  Assert.assertEquals(null,getContentLength(future1));
  Assert.assertEquals(null,getContentLength(future2));
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test public void testPutOpWithRedirect(){
  Future future1=contentLengthFuture(redirectResponse);
  Future future2=contentLengthFuture(errResponse);
  try {
    FSDataOutputStream os=fs.create(p);
    os.write(new byte[]{0});
    os.close();
    Assert.fail();
  }
 catch (  IOException ioe) {
  }
  Assert.assertEquals("0",getContentLength(future1));
  Assert.assertEquals("chunked",getContentLength(future2));
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test public void testDelete(){
  Future future=contentLengthFuture(errResponse);
  try {
    fs.delete(p,false);
    Assert.fail();
  }
 catch (  IOException ioe) {
  }
  Assert.assertEquals(null,getContentLength(future));
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test public void testPutOp(){
  Future future=contentLengthFuture(errResponse);
  try {
    fs.mkdirs(p);
    Assert.fail();
  }
 catch (  IOException ioe) {
  }
  Assert.assertEquals("0",getContentLength(future));
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test public void testPostOp(){
  Future future=contentLengthFuture(errResponse);
  try {
    fs.concat(p,new Path[]{p});
    Assert.fail();
  }
 catch (  IOException ioe) {
  }
  Assert.assertEquals("0",getContentLength(future));
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test public void testPostOpWithRedirect(){
  Future future1=contentLengthFuture(redirectResponse);
  Future future2=contentLengthFuture(errResponse);
  try {
    FSDataOutputStream os=fs.append(p);
    os.write(new byte[]{0});
    os.close();
    Assert.fail();
  }
 catch (  IOException ioe) {
  }
  Assert.assertEquals("0",getContentLength(future1));
  Assert.assertEquals("chunked",getContentLength(future2));
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test public void testGetOp() throws Exception {
  Future future=contentLengthFuture(errResponse);
  try {
    fs.getFileStatus(p);
    Assert.fail();
  }
 catch (  IOException ioe) {
  }
  Assert.assertEquals(null,getContentLength(future));
}

Class: org.apache.hadoop.hdfs.web.TestWebHdfsUrl

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=60000) public void testEncodedPathUrl() throws IOException, URISyntaxException {
  Configuration conf=new Configuration();
  final WebHdfsFileSystem webhdfs=(WebHdfsFileSystem)FileSystem.get(uri,conf);
  String pathName="/hdtest010%2C60020%2C1371000602151.1371058984668";
  Path fsPath=new Path(pathName);
  URL encodedPathUrl=webhdfs.toUrl(PutOpParam.Op.CREATE,fsPath);
  Assert.assertEquals(WebHdfsFileSystem.PATH_PREFIX + pathName,encodedPathUrl.toURI().getPath());
}

Class: org.apache.hadoop.hdfs.web.TestWebHdfsWithMultipleNameNodes

APIUtilityVerifier IterativeVerifier InternalCallVerifier EqualityVerifier 

@Test public void testRedirect() throws Exception {
  final String dir="/testRedirect/";
  final String filename="file";
  final Path p=new Path(dir,filename);
  final String[] writeStrings=createStrings("write to webhdfs ","write");
  final String[] appendStrings=createStrings("append to webhdfs ","append");
  for (int i=0; i < webhdfs.length; i++) {
    final FSDataOutputStream out=webhdfs[i].create(p);
    out.write(writeStrings[i].getBytes());
    out.close();
  }
  for (int i=0; i < webhdfs.length; i++) {
    final long expected=writeStrings[i].length();
    Assert.assertEquals(expected,webhdfs[i].getFileStatus(p).getLen());
  }
  for (int i=0; i < webhdfs.length; i++) {
    final FSDataInputStream in=webhdfs[i].open(p);
    for (int c, j=0; (c=in.read()) != -1; j++) {
      Assert.assertEquals(writeStrings[i].charAt(j),c);
    }
    in.close();
  }
  for (int i=0; i < webhdfs.length; i++) {
    final FSDataOutputStream out=webhdfs[i].append(p);
    out.write(appendStrings[i].getBytes());
    out.close();
  }
  for (int i=0; i < webhdfs.length; i++) {
    final long expected=writeStrings[i].length() + appendStrings[i].length();
    Assert.assertEquals(expected,webhdfs[i].getFileStatus(p).getLen());
  }
  for (int i=0; i < webhdfs.length; i++) {
    final StringBuilder b=new StringBuilder();
    final FSDataInputStream in=webhdfs[i].open(p);
    for (int c; (c=in.read()) != -1; ) {
      b.append((char)c);
    }
    final int wlen=writeStrings[i].length();
    Assert.assertEquals(writeStrings[i],b.substring(0,wlen));
    Assert.assertEquals(appendStrings[i],b.substring(wlen));
    in.close();
  }
}

Class: org.apache.hadoop.hdfs.web.resources.TestParam

APIUtilityVerifier EqualityVerifier 

@Test public void testToSortedStringEscapesURICharacters(){
  final String sep="&";
  Param ampParam=new TokenArgumentParam("token&ersand");
  Param equalParam=new RenewerParam("renewer=equal");
  final String expected="&renewer=renewer%3Dequal&token=token%26ampersand";
  final String actual=Param.toSortedString(sep,equalParam,ampParam);
  Assert.assertEquals(expected,actual);
}

APIUtilityVerifier IterativeVerifier EqualityVerifier 

@Test public void testConcatSourcesParam(){
  final String[] strings={"/","/foo","/bar"};
  for (int n=0; n < strings.length; n++) {
    final String[] sub=new String[n];
    final Path[] paths=new Path[n];
    for (int i=0; i < paths.length; i++) {
      paths[i]=new Path(sub[i]=strings[i]);
    }
    final String expected=StringUtils.join(",",Arrays.asList(sub));
    final ConcatSourcesParam computed=new ConcatSourcesParam(paths);
    Assert.assertEquals(expected,computed.getValue());
  }
}

Class: org.apache.hadoop.http.TestHttpCookieFlag

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testHttpsCookie() throws IOException, GeneralSecurityException {
  URL base=new URL("https://" + NetUtils.getHostPortString(server.getConnectorAddress(1)));
  HttpsURLConnection conn=(HttpsURLConnection)new URL(base,"/echo").openConnection();
  conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory());
  String header=conn.getHeaderField("Set-Cookie");
  List cookies=HttpCookie.parse(header);
  Assert.assertTrue(!cookies.isEmpty());
  Assert.assertTrue(header.contains("; HttpOnly"));
  Assert.assertTrue(cookies.get(0).getSecure());
  Assert.assertTrue("token".equals(cookies.get(0).getValue()));
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testHttpCookie() throws IOException {
  URL base=new URL("http://" + NetUtils.getHostPortString(server.getConnectorAddress(0)));
  HttpURLConnection conn=(HttpURLConnection)new URL(base,"/echo").openConnection();
  String header=conn.getHeaderField("Set-Cookie");
  List cookies=HttpCookie.parse(header);
  Assert.assertTrue(!cookies.isEmpty());
  Assert.assertTrue(header.contains("; HttpOnly"));
  Assert.assertTrue("token".equals(cookies.get(0).getValue()));
}

Class: org.apache.hadoop.http.TestHttpServer

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Verify the administrator access for /logs, /stacks, /conf, /logLevel and
 * /metrics servlets.
 * @throws Exception
 */
@Test public void testAuthorizationOfDefaultServlets() throws Exception {
  Configuration conf=new Configuration();
  conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,true);
  conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_INSTRUMENTATION_REQUIRES_ADMIN,true);
  conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,DummyFilterInitializer.class.getName());
  conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,MyGroupsProvider.class.getName());
  Groups.getUserToGroupsMappingService(conf);
  MyGroupsProvider.clearMapping();
  MyGroupsProvider.mapping.put("userA",Arrays.asList("groupA"));
  MyGroupsProvider.mapping.put("userB",Arrays.asList("groupB"));
  MyGroupsProvider.mapping.put("userC",Arrays.asList("groupC"));
  MyGroupsProvider.mapping.put("userD",Arrays.asList("groupD"));
  MyGroupsProvider.mapping.put("userE",Arrays.asList("groupE"));
  HttpServer2 myServer=new HttpServer2.Builder().setName("test").addEndpoint(new URI("http://localhost:0")).setFindPort(true).setConf(conf).setACL(new AccessControlList("userA,userB groupC,groupD")).build();
  myServer.setAttribute(HttpServer2.CONF_CONTEXT_ATTRIBUTE,conf);
  myServer.start();
  String serverURL="http://" + NetUtils.getHostPortString(myServer.getConnectorAddress(0)) + "/";
  for (  String servlet : new String[]{"conf","logs","stacks","logLevel","metrics"}) {
    for (    String user : new String[]{"userA","userB","userC","userD"}) {
      assertEquals(HttpURLConnection.HTTP_OK,getHttpStatusCode(serverURL + servlet,user));
    }
    assertEquals(HttpURLConnection.HTTP_FORBIDDEN,getHttpStatusCode(serverURL + servlet,"userE"));
  }
  myServer.stop();
}

APIUtilityVerifier EqualityVerifier 

@Test public void testContentTypes() throws Exception {
  URL cssUrl=new URL(baseUrl,"/static/test.css");
  HttpURLConnection conn=(HttpURLConnection)cssUrl.openConnection();
  conn.connect();
  assertEquals(200,conn.getResponseCode());
  assertEquals("text/css",conn.getContentType());
  URL servletUrl=new URL(baseUrl,"/echo?a=b");
  conn=(HttpURLConnection)servletUrl.openConnection();
  conn.connect();
  assertEquals(200,conn.getResponseCode());
  assertEquals("text/plain; charset=utf-8",conn.getContentType());
  servletUrl=new URL(baseUrl,"/echo?a=b.css");
  conn=(HttpURLConnection)servletUrl.openConnection();
  conn.connect();
  assertEquals(200,conn.getResponseCode());
  assertEquals("text/plain; charset=utf-8",conn.getContentType());
  servletUrl=new URL(baseUrl,"/htmlcontent");
  conn=(HttpURLConnection)servletUrl.openConnection();
  conn.connect();
  assertEquals(200,conn.getResponseCode());
  assertEquals("text/html; charset=utf-8",conn.getContentType());
}

APIUtilityVerifier EqualityVerifier 

/** 
 * Test that verifies headers can be up to 64K long. 
 * The test adds a 63K header leaving 1K for other headers.
 * This is because the header buffer setting is for ALL headers,
 * names and values included. 
 */
@Test public void testLongHeader() throws Exception {
  URL url=new URL(baseUrl,"/longheader");
  HttpURLConnection conn=(HttpURLConnection)url.openConnection();
  StringBuilder sb=new StringBuilder();
  for (int i=0; i < 63 * 1024; i++) {
    sb.append("a");
  }
  conn.setRequestProperty("longheader",sb.toString());
  assertEquals(HttpURLConnection.HTTP_OK,conn.getResponseCode());
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testBindAddress() throws Exception {
  checkBindAddress("localhost",0,false).stop();
  HttpServer2 myServer=checkBindAddress("localhost",0,false);
  HttpServer2 myServer2=null;
  try {
    int port=myServer.getConnectorAddress(0).getPort();
    myServer2=checkBindAddress("localhost",port,true);
    port=myServer2.getConnectorAddress(0).getPort();
    myServer2.stop();
    assertNull(myServer2.getConnectorAddress(0));
    myServer2.openListeners();
    assertEquals(port,myServer2.getConnectorAddress(0).getPort());
  }
  finally {
    myServer.stop();
    if (myServer2 != null) {
      myServer2.stop();
    }
  }
}

APIUtilityVerifier EqualityVerifier 

@Test public void testJersey() throws Exception {
  LOG.info("BEGIN testJersey()");
  final String js=readOutput(new URL(baseUrl,"/jersey/foo?op=bar"));
  final Map m=parse(js);
  LOG.info("m=" + m);
  assertEquals("foo",m.get(JerseyResource.PATH));
  assertEquals("bar",m.get(JerseyResource.OP));
  LOG.info("END testJersey()");
}

APIUtilityVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testNoCacheHeader() throws Exception {
  URL url=new URL(baseUrl,"/echo?a=b&c=d");
  HttpURLConnection conn=(HttpURLConnection)url.openConnection();
  assertEquals(HttpURLConnection.HTTP_OK,conn.getResponseCode());
  assertEquals("no-cache",conn.getHeaderField("Cache-Control"));
  assertEquals("no-cache",conn.getHeaderField("Pragma"));
  assertNotNull(conn.getHeaderField("Expires"));
  assertNotNull(conn.getHeaderField("Date"));
  assertEquals(conn.getHeaderField("Expires"),conn.getHeaderField("Date"));
}

Class: org.apache.hadoop.io.TestArrayPrimitiveWritable

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testOldFormat() throws IOException {
  ObjectWritable.writeObject(out,i,i.getClass(),null);
  in.reset(out.getData(),out.getLength());
  @SuppressWarnings("deprecation") String className=UTF8.readString(in);
  assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not labelled as an array of int",i.getClass().getName(),className);
  int length=in.readInt();
  assertEquals("The int[] written by ObjectWritable as a non-compact array " + "was not expected length",i.length,length);
  int[] readValue=new int[length];
  try {
    for (int i=0; i < length; i++) {
      readValue[i]=(int)((Integer)ObjectWritable.readObject(in,null));
    }
  }
 catch (  Exception e) {
    fail("The int[] written by ObjectWritable as a non-compact array " + "was corrupted.  Failed to correctly read int[] of length " + length + ". Got exception:\n"+ StringUtils.stringifyException(e));
  }
  assertTrue("The int[] written by ObjectWritable as a non-compact array " + "was corrupted.",Arrays.equals(i,readValue));
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test @SuppressWarnings("deprecation") public void testObjectLabeling() throws IOException {
  ObjectWritable.writeObject(out,i,i.getClass(),null,true);
  ArrayPrimitiveWritable apw=new ArrayPrimitiveWritable(i);
  ObjectWritable.writeObject(out,apw,apw.getClass(),null,true);
  in.reset(out.getData(),out.getLength());
  String className=UTF8.readString(in);
  assertEquals("The int[] written by ObjectWritable was not labelled as " + "an ArrayPrimitiveWritable.Internal",ArrayPrimitiveWritable.Internal.class.getName(),className);
  ArrayPrimitiveWritable.Internal apwi=new ArrayPrimitiveWritable.Internal();
  apwi.readFields(in);
  assertEquals("The ArrayPrimitiveWritable.Internal component type was corrupted",int.class,apw.getComponentType());
  assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable.Internal was corrupted",Arrays.equals(i,(int[])(apwi.get())));
  String declaredClassName=UTF8.readString(in);
  assertEquals("The APW written by ObjectWritable was not labelled as " + "declaredClass ArrayPrimitiveWritable",ArrayPrimitiveWritable.class.getName(),declaredClassName);
  className=UTF8.readString(in);
  assertEquals("The APW written by ObjectWritable was not labelled as " + "class ArrayPrimitiveWritable",ArrayPrimitiveWritable.class.getName(),className);
  ArrayPrimitiveWritable apw2=new ArrayPrimitiveWritable();
  apw2.readFields(in);
  assertEquals("The ArrayPrimitiveWritable component type was corrupted",int.class,apw2.getComponentType());
  assertTrue("The int[] written by ObjectWritable as " + "ArrayPrimitiveWritable was corrupted",Arrays.equals(i,(int[])(apw2.get())));
}

Class: org.apache.hadoop.io.TestMapFile

APIUtilityVerifier EqualityVerifier 

/** 
 * test {@code MapFile.Reader.midKey() } method 
 */
@Test public void testMidKeyOnCurrentApi() throws Exception {
  final String TEST_PREFIX="testMidKeyOnCurrentApi.mapfile";
  MapFile.Writer writer=null;
  MapFile.Reader reader=null;
  try {
    writer=createWriter(TEST_PREFIX,IntWritable.class,IntWritable.class);
    int SIZE=10;
    for (int i=0; i < SIZE; i++)     writer.append(new IntWritable(i),new IntWritable(i));
    writer.close();
    reader=createReader(TEST_PREFIX,IntWritable.class);
    assertEquals(new IntWritable((SIZE - 1) / 2),reader.midKey());
  }
  finally {
    IOUtils.cleanup(null,writer,reader);
  }
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * test {@code MapFile.Reader.getClosest()} method 
 */
@Test public void testGetClosestOnCurrentApi() throws Exception {
  final String TEST_PREFIX="testGetClosestOnCurrentApi.mapfile";
  MapFile.Writer writer=null;
  MapFile.Reader reader=null;
  try {
    writer=createWriter(TEST_PREFIX,Text.class,Text.class);
    int FIRST_KEY=1;
    for (int i=FIRST_KEY; i < 100; i+=10) {
      Text t=new Text(Integer.toString(i));
      writer.append(t,t);
    }
    writer.close();
    reader=createReader(TEST_PREFIX,Text.class);
    Text key=new Text("55");
    Text value=new Text();
    Text closest=(Text)reader.getClosest(key,value);
    assertEquals(new Text("61"),closest);
    closest=(Text)reader.getClosest(key,value,true);
    assertEquals(new Text("51"),closest);
    final Text explicitKey=new Text("21");
    closest=(Text)reader.getClosest(explicitKey,value);
    assertEquals(new Text("21"),explicitKey);
    key=new Text("00");
    closest=(Text)reader.getClosest(key,value);
    assertEquals(FIRST_KEY,Integer.parseInt(closest.toString()));
    key=new Text("92");
    closest=(Text)reader.getClosest(key,value);
    assertNull("Not null key in testGetClosestWithNewCode",closest);
    closest=(Text)reader.getClosest(key,value,true);
    assertEquals(new Text("91"),closest);
  }
  finally {
    IOUtils.cleanup(null,writer,reader);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test @SuppressWarnings("deprecation") public void testMidKey() throws Exception {
  Path dirName=new Path(TEST_DIR,"testMidKey.mapfile");
  FileSystem fs=FileSystem.getLocal(conf);
  Path qualifiedDirName=fs.makeQualified(dirName);
  MapFile.Writer writer=null;
  MapFile.Reader reader=null;
  try {
    writer=new MapFile.Writer(conf,fs,qualifiedDirName.toString(),IntWritable.class,IntWritable.class);
    writer.append(new IntWritable(1),new IntWritable(1));
    writer.close();
    reader=new MapFile.Reader(qualifiedDirName,conf);
    assertEquals(new IntWritable(1),reader.midKey());
  }
  finally {
    IOUtils.cleanup(null,writer,reader);
  }
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test getClosest feature.
 * @throws Exception
 */
@Test @SuppressWarnings("deprecation") public void testGetClosest() throws Exception {
  Path dirName=new Path(TEST_DIR,"testGetClosest.mapfile");
  FileSystem fs=FileSystem.getLocal(conf);
  Path qualifiedDirName=fs.makeQualified(dirName);
  MapFile.Writer.setIndexInterval(conf,3);
  MapFile.Writer writer=null;
  MapFile.Reader reader=null;
  try {
    writer=new MapFile.Writer(conf,fs,qualifiedDirName.toString(),Text.class,Text.class);
    assertEquals(3,writer.getIndexInterval());
    final int FIRST_KEY=10;
    for (int i=FIRST_KEY; i < 100; i+=10) {
      String iStr=Integer.toString(i);
      Text t=new Text("00".substring(iStr.length()) + iStr);
      writer.append(t,t);
    }
    writer.close();
    reader=new MapFile.Reader(qualifiedDirName,conf);
    Text key=new Text("55");
    Text value=new Text();
    Text closest=(Text)reader.getClosest(key,value);
    assertEquals(new Text("60"),closest);
    closest=(Text)reader.getClosest(key,value,true);
    assertEquals(new Text("50"),closest);
    final Text TWENTY=new Text("20");
    closest=(Text)reader.getClosest(TWENTY,value);
    assertEquals(TWENTY,closest);
    closest=(Text)reader.getClosest(TWENTY,value,true);
    assertEquals(TWENTY,closest);
    key=new Text("00");
    closest=(Text)reader.getClosest(key,value);
    assertEquals(FIRST_KEY,Integer.parseInt(closest.toString()));
    closest=(Text)reader.getClosest(key,value,true);
    assertNull(closest);
    key=new Text("99");
    closest=(Text)reader.getClosest(key,value);
    assertNull(closest);
    closest=(Text)reader.getClosest(key,value,true);
    assertEquals(new Text("90"),closest);
  }
  finally {
    IOUtils.cleanup(null,writer,reader);
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier NullVerifier HybridVerifier 

/** 
 * test all available constructor for {@code MapFile.Writer}
 */
@Test @SuppressWarnings("deprecation") public void testDeprecatedConstructors(){
  String path=new Path(TEST_DIR,"writes.mapfile").toString();
  MapFile.Writer writer=null;
  MapFile.Reader reader=null;
  try {
    FileSystem fs=FileSystem.getLocal(conf);
    writer=new MapFile.Writer(conf,fs,path,IntWritable.class,Text.class,CompressionType.RECORD);
    assertNotNull(writer);
    writer.close();
    writer=new MapFile.Writer(conf,fs,path,IntWritable.class,Text.class,CompressionType.RECORD,defaultProgressable);
    assertNotNull(writer);
    writer.close();
    writer=new MapFile.Writer(conf,fs,path,IntWritable.class,Text.class,CompressionType.RECORD,defaultCodec,defaultProgressable);
    assertNotNull(writer);
    writer.close();
    writer=new MapFile.Writer(conf,fs,path,WritableComparator.get(Text.class),Text.class);
    assertNotNull(writer);
    writer.close();
    writer=new MapFile.Writer(conf,fs,path,WritableComparator.get(Text.class),Text.class,SequenceFile.CompressionType.RECORD);
    assertNotNull(writer);
    writer.close();
    writer=new MapFile.Writer(conf,fs,path,WritableComparator.get(Text.class),Text.class,CompressionType.RECORD,defaultProgressable);
    assertNotNull(writer);
    writer.close();
    reader=new MapFile.Reader(fs,path,WritableComparator.get(IntWritable.class),conf);
    assertNotNull(reader);
    assertNotNull("reader key is null !!!",reader.getKeyClass());
    assertNotNull("reader value in null",reader.getValueClass());
  }
 catch (  IOException e) {
    fail(e.getMessage());
  }
 finally {
    IOUtils.cleanup(null,writer,reader);
  }
}

APIUtilityVerifier IterativeVerifier UtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier HybridVerifier 

/** 
 * test {@code MapFile.Reader.next(key, value)} for iteration.
 */
@Test public void testReaderKeyIteration(){
  final String TEST_METHOD_KEY="testReaderKeyIteration.mapfile";
  int SIZE=10;
  int ITERATIONS=5;
  MapFile.Writer writer=null;
  MapFile.Reader reader=null;
  try {
    writer=createWriter(TEST_METHOD_KEY,IntWritable.class,Text.class);
    int start=0;
    for (int i=0; i < SIZE; i++)     writer.append(new IntWritable(i),new Text("Value:" + i));
    writer.close();
    reader=createReader(TEST_METHOD_KEY,IntWritable.class);
    Writable startValue=new Text("Value:" + start);
    int i=0;
    while (i++ < ITERATIONS) {
      IntWritable key=new IntWritable(start);
      Writable value=startValue;
      while (reader.next(key,value)) {
        assertNotNull(key);
        assertNotNull(value);
      }
      reader.reset();
    }
    assertTrue("reader seek error !!!",reader.seek(new IntWritable(SIZE / 2)));
    assertFalse("reader seek error !!!",reader.seek(new IntWritable(SIZE * 2)));
  }
 catch (  IOException ex) {
    fail("reader seek error !!!");
  }
 finally {
    IOUtils.cleanup(null,writer,reader);
  }
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test @SuppressWarnings("deprecation") public void testMidKeyEmpty() throws Exception {
  Path dirName=new Path(TEST_DIR,"testMidKeyEmpty.mapfile");
  FileSystem fs=FileSystem.getLocal(conf);
  Path qualifiedDirName=fs.makeQualified(dirName);
  MapFile.Writer writer=new MapFile.Writer(conf,fs,qualifiedDirName.toString(),IntWritable.class,IntWritable.class);
  writer.close();
  MapFile.Reader reader=new MapFile.Reader(qualifiedDirName,conf);
  try {
    assertEquals(null,reader.midKey());
  }
  finally {
    reader.close();
  }
}

APIUtilityVerifier BranchVerifier UtilityVerifier BooleanVerifier HybridVerifier 

/** 
 * test {@code MapFile.Writer.testFix} method
 */
@Test public void testFix(){
  final String INDEX_LESS_MAP_FILE="testFix.mapfile";
  int PAIR_SIZE=20;
  MapFile.Writer writer=null;
  try {
    FileSystem fs=FileSystem.getLocal(conf);
    Path dir=new Path(TEST_DIR,INDEX_LESS_MAP_FILE);
    writer=createWriter(INDEX_LESS_MAP_FILE,IntWritable.class,Text.class);
    for (int i=0; i < PAIR_SIZE; i++)     writer.append(new IntWritable(0),new Text("value"));
    writer.close();
    File indexFile=new File(".","." + INDEX_LESS_MAP_FILE + "/index");
    boolean isDeleted=false;
    if (indexFile.exists())     isDeleted=indexFile.delete();
    if (isDeleted)     assertTrue("testFix error !!!",MapFile.fix(fs,dir,IntWritable.class,Text.class,true,conf) == PAIR_SIZE);
  }
 catch (  Exception ex) {
    fail("testFix error !!!");
  }
 finally {
    IOUtils.cleanup(null,writer);
  }
}

Class: org.apache.hadoop.io.compress.TestCodec

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testGzipCodecRead() throws IOException {
  Configuration conf=new Configuration();
  conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,false);
  assertFalse("ZlibFactory is using native libs against request",ZlibFactory.isNativeZlibLoaded(conf));
  Decompressor zlibDecompressor=ZlibFactory.getZlibDecompressor(conf);
  assertNotNull("zlibDecompressor is null!",zlibDecompressor);
  assertTrue("ZlibFactory returned unexpected inflator",zlibDecompressor instanceof BuiltInZlibInflater);
  CodecPool.returnDecompressor(zlibDecompressor);
  String tmpDir=System.getProperty("test.build.data","/tmp/");
  Path f=new Path(new Path(tmpDir),"testGzipCodecRead.txt.gz");
  BufferedWriter bw=new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(f.toString()))));
  final String msg="This is the message in the file!";
  bw.write(msg);
  bw.close();
  CompressionCodecFactory ccf=new CompressionCodecFactory(conf);
  CompressionCodec codec=ccf.getCodec(f);
  Decompressor decompressor=CodecPool.getDecompressor(codec);
  FileSystem fs=FileSystem.getLocal(conf);
  InputStream is=fs.open(f);
  is=codec.createInputStream(is,decompressor);
  BufferedReader br=new BufferedReader(new InputStreamReader(is));
  String line=br.readLine();
  assertEquals("Didn't get the same message back!",msg,line);
  br.close();
}

APIUtilityVerifier InternalCallVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testGzipCompatibility() throws IOException {
  Random r=new Random();
  long seed=r.nextLong();
  r.setSeed(seed);
  LOG.info("seed: " + seed);
  DataOutputBuffer dflbuf=new DataOutputBuffer();
  GZIPOutputStream gzout=new GZIPOutputStream(dflbuf);
  byte[] b=new byte[r.nextInt(128 * 1024 + 1)];
  r.nextBytes(b);
  gzout.write(b);
  gzout.close();
  DataInputBuffer gzbuf=new DataInputBuffer();
  gzbuf.reset(dflbuf.getData(),dflbuf.getLength());
  Configuration conf=new Configuration();
  conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,false);
  CompressionCodec codec=ReflectionUtils.newInstance(GzipCodec.class,conf);
  Decompressor decom=codec.createDecompressor();
  assertNotNull(decom);
  assertEquals(BuiltInGzipDecompressor.class,decom.getClass());
  InputStream gzin=codec.createInputStream(gzbuf,decom);
  dflbuf.reset();
  IOUtils.copyBytes(gzin,dflbuf,4096);
  final byte[] dflchk=Arrays.copyOf(dflbuf.getData(),dflbuf.getLength());
  assertArrayEquals(b,dflchk);
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier 

@Test public void testCodecPoolGzipReuse() throws Exception {
  Configuration conf=new Configuration();
  conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,true);
  if (!ZlibFactory.isNativeZlibLoaded(conf)) {
    LOG.warn("testCodecPoolGzipReuse skipped: native libs not loaded");
    return;
  }
  GzipCodec gzc=ReflectionUtils.newInstance(GzipCodec.class,conf);
  DefaultCodec dfc=ReflectionUtils.newInstance(DefaultCodec.class,conf);
  Compressor c1=CodecPool.getCompressor(gzc);
  Compressor c2=CodecPool.getCompressor(dfc);
  CodecPool.returnCompressor(c1);
  CodecPool.returnCompressor(c2);
  assertTrue("Got mismatched ZlibCompressor",c2 != CodecPool.getCompressor(gzc));
}

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test public void testGzipLongOverflow() throws IOException {
  LOG.info("testGzipLongOverflow");
  Configuration conf=new Configuration();
  conf.setBoolean(CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY,false);
  assertFalse("ZlibFactory is using native libs against request",ZlibFactory.isNativeZlibLoaded(conf));
  Decompressor zlibDecompressor=ZlibFactory.getZlibDecompressor(conf);
  assertNotNull("zlibDecompressor is null!",zlibDecompressor);
  assertTrue("ZlibFactory returned unexpected inflator",zlibDecompressor instanceof BuiltInZlibInflater);
  CodecPool.returnDecompressor(zlibDecompressor);
  String tmpDir=System.getProperty("test.build.data","/tmp/");
  Path f=new Path(new Path(tmpDir),"testGzipLongOverflow.bin.gz");
  BufferedWriter bw=new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(f.toString()))));
  final int NBUF=1024 * 4 + 1;
  final char[] buf=new char[1024 * 1024];
  for (int i=0; i < buf.length; i++)   buf[i]='\0';
  for (int i=0; i < NBUF; i++) {
    bw.write(buf);
  }
  bw.close();
  CompressionCodecFactory ccf=new CompressionCodecFactory(conf);
  CompressionCodec codec=ccf.getCodec(f);
  Decompressor decompressor=CodecPool.getDecompressor(codec);
  FileSystem fs=FileSystem.getLocal(conf);
  InputStream is=fs.open(f);
  is=codec.createInputStream(is,decompressor);
  BufferedReader br=new BufferedReader(new InputStreamReader(is));
  for (int j=0; j < NBUF; j++) {
    int n=br.read(buf);
    assertEquals("got wrong read length!",n,buf.length);
    for (int i=0; i < buf.length; i++)     assertEquals("got wrong byte!",buf[i],'\0');
  }
  br.close();
}

Class: org.apache.hadoop.io.compress.lz4.TestLz4CompressorDecompressor

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testCompressDecompress(){
  int BYTE_SIZE=1024 * 54;
  byte[] bytes=generate(BYTE_SIZE);
  Lz4Compressor compressor=new Lz4Compressor();
  try {
    compressor.setInput(bytes,0,bytes.length);
    assertTrue("Lz4CompressDecompress getBytesRead error !!!",compressor.getBytesRead() > 0);
    assertTrue("Lz4CompressDecompress getBytesWritten before compress error !!!",compressor.getBytesWritten() == 0);
    byte[] compressed=new byte[BYTE_SIZE];
    int cSize=compressor.compress(compressed,0,compressed.length);
    assertTrue("Lz4CompressDecompress getBytesWritten after compress error !!!",compressor.getBytesWritten() > 0);
    Lz4Decompressor decompressor=new Lz4Decompressor();
    decompressor.setInput(compressed,0,cSize);
    byte[] decompressed=new byte[BYTE_SIZE];
    decompressor.decompress(decompressed,0,decompressed.length);
    assertTrue("testLz4CompressDecompress finished error !!!",decompressor.finished());
    assertArrayEquals(bytes,decompressed);
    compressor.reset();
    decompressor.reset();
    assertTrue("decompressor getRemaining error !!!",decompressor.getRemaining() == 0);
  }
 catch (  Exception e) {
    fail("testLz4CompressDecompress ex error!!!");
  }
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test public void testCompressorDecopressorLogicWithCompressionStreams(){
  DataOutputStream deflateOut=null;
  DataInputStream inflateIn=null;
  int BYTE_SIZE=1024 * 100;
  byte[] bytes=generate(BYTE_SIZE);
  int bufferSize=262144;
  int compressionOverhead=(bufferSize / 6) + 32;
  try {
    DataOutputBuffer compressedDataBuffer=new DataOutputBuffer();
    CompressionOutputStream deflateFilter=new BlockCompressorStream(compressedDataBuffer,new Lz4Compressor(bufferSize),bufferSize,compressionOverhead);
    deflateOut=new DataOutputStream(new BufferedOutputStream(deflateFilter));
    deflateOut.write(bytes,0,bytes.length);
    deflateOut.flush();
    deflateFilter.finish();
    DataInputBuffer deCompressedDataBuffer=new DataInputBuffer();
    deCompressedDataBuffer.reset(compressedDataBuffer.getData(),0,compressedDataBuffer.getLength());
    CompressionInputStream inflateFilter=new BlockDecompressorStream(deCompressedDataBuffer,new Lz4Decompressor(bufferSize),bufferSize);
    inflateIn=new DataInputStream(new BufferedInputStream(inflateFilter));
    byte[] result=new byte[BYTE_SIZE];
    inflateIn.read(result);
    assertArrayEquals("original array not equals compress/decompressed array",result,bytes);
  }
 catch (  IOException e) {
    fail("testLz4CompressorDecopressorLogicWithCompressionStreams ex error !!!");
  }
 finally {
    try {
      if (deflateOut != null)       deflateOut.close();
      if (inflateIn != null)       inflateIn.close();
    }
 catch (    Exception e) {
    }
  }
}

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier PublicFieldVerifier HybridVerifier 

@Test public void testSetInputWithBytesSizeMoreThenDefaultLz4CompressorByfferSize(){
  int BYTES_SIZE=1024 * 64 + 1;
  try {
    Lz4Compressor compressor=new Lz4Compressor();
    byte[] bytes=generate(BYTES_SIZE);
    assertTrue("needsInput error !!!",compressor.needsInput());
    compressor.setInput(bytes,0,bytes.length);
    byte[] emptyBytes=new byte[BYTES_SIZE];
    int csize=compressor.compress(emptyBytes,0,bytes.length);
    assertTrue("testSetInputWithBytesSizeMoreThenDefaultLz4CompressorByfferSize error !!!",csize != 0);
  }
 catch (  Exception ex) {
    fail("testSetInputWithBytesSizeMoreThenDefaultLz4CompressorByfferSize ex error !!!");
  }
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test public void testCompressorDecompressorEmptyStreamLogic(){
  ByteArrayInputStream bytesIn=null;
  ByteArrayOutputStream bytesOut=null;
  byte[] buf=null;
  BlockDecompressorStream blockDecompressorStream=null;
  try {
    bytesOut=new ByteArrayOutputStream();
    BlockCompressorStream blockCompressorStream=new BlockCompressorStream(bytesOut,new Lz4Compressor(),1024,0);
    blockCompressorStream.close();
    buf=bytesOut.toByteArray();
    assertEquals("empty stream compressed output size != 4",4,buf.length);
    bytesIn=new ByteArrayInputStream(buf);
    blockDecompressorStream=new BlockDecompressorStream(bytesIn,new Lz4Decompressor(),1024);
    assertEquals("return value is not -1",-1,blockDecompressorStream.read());
  }
 catch (  Exception e) {
    fail("testCompressorDecompressorEmptyStreamLogic ex error !!!" + e.getMessage());
  }
 finally {
    if (blockDecompressorStream != null)     try {
      bytesIn.close();
      bytesOut.close();
      blockDecompressorStream.close();
    }
 catch (    IOException e) {
    }
  }
}

Class: org.apache.hadoop.io.compress.snappy.TestSnappyCompressorDecompressor

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test public void testCompressorDecompressorEmptyStreamLogic(){
  ByteArrayInputStream bytesIn=null;
  ByteArrayOutputStream bytesOut=null;
  byte[] buf=null;
  BlockDecompressorStream blockDecompressorStream=null;
  try {
    bytesOut=new ByteArrayOutputStream();
    BlockCompressorStream blockCompressorStream=new BlockCompressorStream(bytesOut,new SnappyCompressor(),1024,0);
    blockCompressorStream.close();
    buf=bytesOut.toByteArray();
    assertEquals("empty stream compressed output size != 4",4,buf.length);
    bytesIn=new ByteArrayInputStream(buf);
    blockDecompressorStream=new BlockDecompressorStream(bytesIn,new SnappyDecompressor(),1024);
    assertEquals("return value is not -1",-1,blockDecompressorStream.read());
  }
 catch (  Exception e) {
    fail("testCompressorDecompressorEmptyStreamLogic ex error !!!" + e.getMessage());
  }
 finally {
    if (blockDecompressorStream != null)     try {
      bytesIn.close();
      bytesOut.close();
      blockDecompressorStream.close();
    }
 catch (    IOException e) {
    }
  }
}

Class: org.apache.hadoop.io.compress.zlib.TestZlibCompressorDecompressor

APIUtilityVerifier UtilityVerifier InternalCallVerifier BooleanVerifier EqualityVerifier HybridVerifier 

@Test public void testZlibCompressDecompress(){
  byte[] rawData=null;
  int rawDataSize=0;
  rawDataSize=1024 * 64;
  rawData=generate(rawDataSize);
  try {
    ZlibCompressor compressor=new ZlibCompressor();
    ZlibDecompressor decompressor=new ZlibDecompressor();
    assertFalse("testZlibCompressDecompress finished error",compressor.finished());
    compressor.setInput(rawData,0,rawData.length);
    assertTrue("testZlibCompressDecompress getBytesRead before error",compressor.getBytesRead() == 0);
    compressor.finish();
    byte[] compressedResult=new byte[rawDataSize];
    int cSize=compressor.compress(compressedResult,0,rawDataSize);
    assertTrue("testZlibCompressDecompress getBytesRead ather error",compressor.getBytesRead() == rawDataSize);
    assertTrue("testZlibCompressDecompress compressed size no less then original size",cSize < rawDataSize);
    decompressor.setInput(compressedResult,0,cSize);
    byte[] decompressedBytes=new byte[rawDataSize];
    decompressor.decompress(decompressedBytes,0,decompressedBytes.length);
    assertArrayEquals("testZlibCompressDecompress arrays not equals ",rawData,decompressedBytes);
    compressor.reset();
    decompressor.reset();
  }
 catch (  IOException ex) {
    fail("testZlibCompressDecompress ex !!!" + ex);
  }
}

Class: org.apache.hadoop.io.file.tfile.TestTFileByteArrays

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test public void testLocate() throws IOException {
  if (skip)   return;
  writeRecords(3 * records1stBlock);
  Reader reader=new Reader(fs.open(path),fs.getFileStatus(path).getLen(),conf);
  Scanner scanner=reader.createScanner();
  locate(scanner,composeSortedKey(KEY,2).getBytes());
  locate(scanner,composeSortedKey(KEY,records1stBlock - 1).getBytes());
  locate(scanner,composeSortedKey(KEY,records1stBlock).getBytes());
  Location locX=locate(scanner,"keyX".getBytes());
  Assert.assertEquals(scanner.endLocation,locX);
  scanner.close();
  reader.close();
}

Class: org.apache.hadoop.io.nativeio.TestNativeIO

APIUtilityVerifier IterativeVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

/** 
 * Test for races in fstat usage
 * NOTE: this test is likely to fail on RHEL 6.0 which has a non-threadsafe
 * implementation of getpwuid_r.
 */
@Test(timeout=30000) public void testMultiThreadedFstat() throws Exception {
  if (Path.WINDOWS) {
    return;
  }
  final FileOutputStream fos=new FileOutputStream(new File(TEST_DIR,"testfstat"));
  final AtomicReference thrown=new AtomicReference();
  List statters=new ArrayList();
  for (int i=0; i < 10; i++) {
    Thread statter=new Thread(){
      @Override public void run(){
        long et=Time.now() + 5000;
        while (Time.now() < et) {
          try {
            NativeIO.POSIX.Stat stat=NativeIO.POSIX.getFstat(fos.getFD());
            assertEquals(System.getProperty("user.name"),stat.getOwner());
            assertNotNull(stat.getGroup());
            assertTrue(!stat.getGroup().isEmpty());
            assertEquals("Stat mode field should indicate a regular file",NativeIO.POSIX.Stat.S_IFREG,stat.getMode() & NativeIO.POSIX.Stat.S_IFMT);
          }
 catch (          Throwable t) {
            thrown.set(t);
          }
        }
      }
    }
;
    statters.add(statter);
    statter.start();
  }
  for (  Thread t : statters) {
    t.join();
  }
  fos.close();
  if (thrown.get() != null) {
    throw new RuntimeException(thrown.get());
  }
}

APIUtilityVerifier AssumptionSetter EqualityVerifier HybridVerifier 

@Test(timeout=10000) public void testMlock() throws Exception {
  assumeTrue(NativeIO.isAvailable());
  final File TEST_FILE=new File(new File(System.getProperty("test.build.data","build/test/data")),"testMlockFile");
  final int BUF_LEN=12289;
  byte buf[]=new byte[BUF_LEN];
  int bufSum=0;
  for (int i=0; i < buf.length; i++) {
    buf[i]=(byte)(i % 60);
    bufSum+=buf[i];
  }
  FileOutputStream fos=new FileOutputStream(TEST_FILE);
  try {
    fos.write(buf);
    fos.getChannel().force(true);
  }
  finally {
    fos.close();
  }
  FileInputStream fis=null;
  FileChannel channel=null;
  try {
    fis=new FileInputStream(TEST_FILE);
    channel=fis.getChannel();
    long fileSize=channel.size();
    MappedByteBuffer mapbuf=channel.map(MapMode.READ_ONLY,0,fileSize);
    NativeIO.POSIX.mlock(mapbuf,fileSize);
    int sum=0;
    for (int i=0; i < fileSize; i++) {
      sum+=mapbuf.get(i);
    }
    assertEquals("Expected sums to be equal",bufSum,sum);
    NativeIO.POSIX.munmap(mapbuf);
  }
  finally {
    if (channel != null) {
      channel.close();
    }
    if (fis != null) {
      fis.close();
    }
  }
}

APIUtilityVerifier UtilityVerifier BooleanVerifier HybridVerifier 

@Test(timeout=30000) public void testSetFilePointer() throws Exception {
  if (!Path.WINDOWS) {
    return;
  }
  LOG.info("Set a file pointer on Windows");
  try {
    File testfile=new File(TEST_DIR,"testSetFilePointer");
    assertTrue("Create test subject",testfile.exists() || testfile.createNewFile());
    FileWriter writer=new FileWriter(testfile);
    try {
      for (int i=0; i < 200; i++)       if (i < 100)       writer.write('a');
 else       writer.write('b');
      writer.flush();
    }
 catch (    Exception writerException) {
      fail("Got unexpected exception: " + writerException.getMessage());
    }
 finally {
      writer.close();
    }
    FileDescriptor fd=NativeIO.Windows.createFile(testfile.getCanonicalPath(),NativeIO.Windows.GENERIC_READ,NativeIO.Windows.FILE_SHARE_READ | NativeIO.Windows.FILE_SHARE_WRITE | NativeIO.Windows.FILE_SHARE_DELETE,NativeIO.Windows.OPEN_EXISTING);
    NativeIO.Windows.setFilePointer(fd,120,NativeIO.Windows.FILE_BEGIN);
    FileReader reader=new FileReader(fd);
    try {
      int c=reader.read();
      assertTrue("Unexpected character: " + c,c == 'b');
    }
 catch (    Exception readerException) {
      fail("Got unexpected exception: " + readerException.getMessage());
    }
 finally {
      reader.close();
    }
  }
 catch (  Exception e) {
    fail("Got unexpected exception: " + e.getMessage());
  }
}

APIUtilityVerifier UtilityVerifier BooleanVerifier HybridVerifier 

@Test(timeout=30000) public void testCreateFile() throws Exception {
  if (!Path.WINDOWS) {
    return;
  }
  LOG.info("Open a file on Windows with SHARE_DELETE shared mode");
  try {
    File testfile=new File(TEST_DIR,"testCreateFile");
    assertTrue("Create test subject",testfile.exists() || testfile.createNewFile());
    FileDescriptor fd=NativeIO.Windows.createFile(testfile.getCanonicalPath(),NativeIO.Windows.GENERIC_READ,NativeIO.Windows.FILE_SHARE_READ | NativeIO.Windows.FILE_SHARE_WRITE | NativeIO.Windows.FILE_SHARE_DELETE,NativeIO.Windows.OPEN_EXISTING);
    FileInputStream fin=new FileInputStream(fd);
    try {
      fin.read();
      File newfile=new File(TEST_DIR,"testRenamedFile");
      boolean renamed=testfile.renameTo(newfile);
      assertTrue("Rename failed.",renamed);
      fin.read();
    }
 catch (    Exception e) {
      fail("Got unexpected exception: " + e.getMessage());
    }
 finally {
      fin.close();
    }
  }
 catch (  Exception e) {
    fail("Got unexpected exception: " + e.getMessage());
  }
}

APIUtilityVerifier InternalCallVerifier BooleanVerifier NullVerifier EqualityVerifier HybridVerifier 

@Test(timeout=30000) public void testFstat() throws Exception {
  FileOutputStream fos=new FileOutputStream(new File(TEST_DIR,"testfstat"));
  NativeIO.POSIX.Stat stat=NativeIO.POSIX.getFstat(fos.getFD());
  fos.close();
  LOG.info("Stat: " + String.valueOf(stat));
  String owner=stat.getOwner();
  String expectedOwner=System.getProperty("user.name");
  if (Path.WINDOWS) {
    UserGroupInformation ugi=UserGroupInformation.createRemoteUser(expectedOwner);
    final String adminsGroupString="Administrators";
    if (Arrays.asList(ugi.getGroupNames()).contains(adminsGroupString)) {
      expectedOwner=adminsGroupString;
    }
  }
  assertEquals(expectedOwner,owner);
  assertNotNull(stat.getGroup());
  assertTrue(!stat.getGroup().isEmpty());
  assertEquals("Stat mode field should indicate a regular file",NativeIO.POSIX.Stat.S_IFREG,stat.getMode() & NativeIO.POSIX.Stat.S_IFMT);
}

Class: org.apache.hadoop.io.nativeio.TestSharedFileDescriptorFactory

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

@Test(timeout=10000) public void testReadAndWrite() throws Exception {
  File path=new File(TEST_BASE,"testReadAndWrite");
  path.mkdirs();
  SharedFileDescriptorFactory factory=SharedFileDescriptorFactory.create("woot_",new String[]{path.getAbsolutePath()});
  FileInputStream inStream=factory.createDescriptor("testReadAndWrite",4096);
  FileOutputStream outStream=new FileOutputStream(inStream.getFD());
  outStream.write(101);
  inStream.getChannel().position(0);
  Assert.assertEquals(101,inStream.read());
  inStream.close();
  outStream.close();
  FileUtil.fullyDelete(path);
}

APIUtilityVerifier BooleanVerifier AssumptionSetter HybridVerifier 

@Test(timeout=10000) public void testCleanupRemainders() throws Exception {
  Assume.assumeTrue(NativeIO.isAvailable());
  Assume.assumeTrue(SystemUtils.IS_OS_UNIX);
  File path=new File(TEST_BASE,"testCleanupRemainders");
  path.mkdirs();
  String remainder1=path.getAbsolutePath() + Path.SEPARATOR + "woot2_remainder1";
  String remainder2=path.getAbsolutePath() + Path.SEPARATOR + "woot2_remainder2";
  createTempFile(remainder1);
  createTempFile(remainder2);
  SharedFileDescriptorFactory.create("woot2_",new String[]{path.getAbsolutePath()});
  Assert.assertFalse(new File(remainder1).exists());
  Assert.assertFalse(new File(remainder2).exists());
  FileUtil.fullyDelete(path);
}

APIUtilityVerifier UtilityVerifier EqualityVerifier HybridVerifier 

@Test(timeout=60000) public void testDirectoryFallbacks() throws Exception {
  File nonExistentPath=new File(TEST_BASE,"nonexistent");
  File permissionDeniedPath=new File("/");
  File goodPath=new File(TEST_BASE,"testDirectoryFallbacks");
  goodPath.mkdirs();
  try {
    SharedFileDescriptorFactory.create("shm_",new String[]{nonExistentPath.getAbsolutePath(),permissionDeniedPath.getAbsolutePath()});
    Assert.fail();
  }
 catch (  IOException e) {
  }
  SharedFileDescriptorFactory factory=SharedFileDescriptorFactory.create("shm_",new String[]{nonExistentPath.getAbsolutePath(),permissionDeniedPath.getAbsolutePath(),goodPath.getAbsolutePath()});
  Assert.assertEquals(goodPath.getAbsolutePath(),factory.getPath());
  FileUtil.fullyDelete(goodPath);
}

Class: org.apache.hadoop.io.retry.TestFailoverProxy

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Test that concurrent failed method invocations only result in a single
 * failover.
 */
@Test public void testConcurrentMethodFailures() throws InterruptedException {
  FlipFlopProxyProvider proxyProvider=new FlipFlopProxyProvider(UnreliableInterface.class,new SynchronizedUnreliableImplementation("impl1",TypeOfExceptionToFailWith.STANDBY_EXCEPTION,2),new UnreliableImplementation("impl2",TypeOfExceptionToFailWith.STANDBY_EXCEPTION));
  final UnreliableInterface unreliable=(UnreliableInterface)RetryProxy.create(UnreliableInterface.class,proxyProvider,RetryPolicies.failoverOnNetworkException(10));
  ConcurrentMethodThread t1=new ConcurrentMethodThread(unreliable);
  ConcurrentMethodThread t2=new ConcurrentMethodThread(unreliable);
  t1.start();
  t2.start();
  t1.join();
  t2.join();
  assertEquals("impl2",t1.result);
  assertEquals("impl2",t2.result);
  assertEquals(1,proxyProvider.getFailoversOccurred());
}

APIUtilityVerifier InternalCallVerifier EqualityVerifier 

/** 
 * Ensure that when all configured services are throwing StandbyException
 * that we fail over back and forth between them until one is no longer
 * throwing StandbyException.
 */
@Test public void testFailoverBetweenMultipleStandbys() throws UnreliableException, StandbyException, IOException {
  final long millisToSleep=10000;
  final UnreliableImplementation impl1=new UnreliableImplementation("impl1",TypeOfExceptionToFailWith.STANDBY_EXCEPTION);
  FlipFlopProxyProvider proxyProvider=new FlipFlopProxyProvider(UnreliableInterface.class,impl1,new UnreliableImplementation("impl2",TypeOfExceptionToFailWith.STANDBY_EXCEPTION));
  final UnreliableInterface unreliable=(UnreliableInterface)RetryProxy.create(UnreliableInterface.class,proxyProvider,RetryPolicies.failoverOnNetworkException(RetryPolicies.TRY_ONCE_THEN_FAIL,10,1000,10000));
  new Thread(){
    @Override public void run(){
      ThreadUtil.sleepAtLeastIgnoreInterrupts(millisToSleep);
      impl1.setIdentifier("renamed-impl1");
    }
  }
.start();
  String result=unreliable.failsIfIdentifierDoesntMatch("renamed-impl1");
  assertEquals("renamed-impl1",result);
}